framework,version,device,op_name,kernel_source,gemm_dtype,m,n,k,latency
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,8192,18.54018825954861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,10240,23.022271050347225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,12288,27.407350328233505
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,8192,31.982309977213543
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,16384,35.87409803602431
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,10240,39.85551622178819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,7168,16.604219224717884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,12288,47.958709716796875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,6144,14.42176988389757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,7168,27.636427137586807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,6144,24.33437771267361
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,5120,12.425074259440104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,5120,19.894783867730034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,16384,61.76958550347222
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,4096,10.301940070258246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,4096,15.926873101128471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,3584,9.203943040635851
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,3072,8.82876671685113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,3584,14.023674011230469
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,3072,12.169757419162325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,2560,9.98366461859809
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,2048,6.450211418999566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,1536,6.097404479980469
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,2048,8.033516777886286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,2560,7.746795654296875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,256,1.9173403845893011
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,512,2.846055136786567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,1536,5.345580630832249
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,1024,4.313121795654297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,1024,5.549758063422309
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,768,3.912705739339193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,768,4.126622941758898
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,512,3.5411084493001304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,128,2.1779600779215493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,64,2.090740415785048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,128,2.821520063612196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,32,2.3573190901014542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,256,3.0251653459337025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,64,2.5733990139431424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,32,2.6006240844726562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,10240,8.209968566894531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,10240,10.065311855740017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,12288,10.002017550998264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,12288,12.160282558865019
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,16384,12.571400960286459
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,16384,16.25298902723524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,8192,7.958861456976996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,8192,6.5282499525282125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,7168,7.374529520670573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,7168,6.098900265163845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,6144,5.966525183783637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,6144,5.291982014973958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,5120,4.334180619981554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,4096,3.7619751824273
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,4096,5.480801900227864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,5120,6.537509494357639
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,3584,4.805774264865451
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,3072,3.0811920166015625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,3584,3.287025875515408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,3072,2.892599953545464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,2048,2.0173867543538413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,2560,2.485818650987413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,2560,2.4463698069254556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,1536,1.773154682583279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,2048,1.9240275488959417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,1024,1.4441324869791667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,1024,1.218024041917589
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,1536,1.5380658043755426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,768,1.1883209016587999
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,512,0.6886720127529569
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,768,1.1570587158203125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,256,0.4664230876498752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,128,0.4209706783294678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,64,0.38095111317104763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,256,0.7920462290445963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,128,0.8686168988545736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,512,0.9825057983398438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,32,0.3769377867380778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,64,0.6893573337131076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,32,0.6687111324734158
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,12288,8.383146497938368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,12288,8.991798400878906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,16384,10.797016567654081
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,16384,12.046329074435763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,10240,7.081020355224609
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,10240,7.669706556532119
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,8192,5.849101172553168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,8192,8.857780456542969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,7168,5.267770555284288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,7168,5.080331590440538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,6144,4.882584042019315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,65536,52.67180379231771
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,6144,4.4548259311252165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,5120,3.7752808464898
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,5120,3.7583465576171875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,4096,3.1039839850531687
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,4096,4.46018303765191
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,3584,3.6899163987901478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,3584,2.7426026662190757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,3072,2.5435013241238065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,3072,2.364915635850694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,2560,2.71650759379069
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,2048,1.670749346415202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,2560,2.0344986385769315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,2048,1.5428924560546875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,65536,61.442654079861114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,1024,1.0698489083184135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,1536,1.3044186698065865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,1536,1.2295217514038086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,512,0.589063114590115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,1024,0.9438773261176215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,768,0.8806008762783475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,512,0.7247991032070584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,256,0.3495351208580865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,768,0.9171466827392578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,128,0.2874853346082899
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,64,0.28214311599731445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,256,0.6038933330112034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,32,0.30192444059583873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,128,0.5426586469014486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,64,0.5173679987589518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,32,0.6311768955654568
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,12288,7.55636003282335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,16384,10.049624972873264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,12288,7.600513034396702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,16384,9.973684522840712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,65536,45.102820502387154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,65536,46.77448187934028
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,10240,6.276197221544053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,8192,5.278195699055989
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,10240,6.473067389594184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,8192,7.492951287163629
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,7168,5.98831303914388
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,7168,4.609212239583333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,6144,4.922530704074435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,6144,4.0518472459581165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,5120,4.0157652960883246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,5120,3.3597128126356335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,4096,2.8732115427652993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,4096,3.66755379570855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,3584,3.0645618438720703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,3584,2.465956370035807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,2560,1.8128276401095922
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,3072,2.664465798272027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,3072,2.1366151173909507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,2560,1.835831112331814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,2048,1.6424747043185766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,1024,0.7346675660875109
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,2048,1.3655839496188695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,1536,1.266118261549208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,1536,1.1090631484985352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,1024,0.8142648802863227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,768,0.7186435593499078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,512,0.4038879871368408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,256,0.24842577510409883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,128,0.2371048927307129
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,64,0.23656090100606283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,512,0.6877911355760363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,256,0.4996444384256999
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,768,0.768215126461453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,128,0.5136853324042426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,32,0.23862576484680176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,64,0.5013502438863119
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,32,0.45191913180881077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,12288,6.0218700832790795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,12288,7.4493255615234375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,16384,8.075309753417969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,16384,9.80599127875434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,10240,5.01400883992513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,65536,38.77087741427952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,8192,5.5265990363226996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,10240,6.244004567464192
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,7168,3.5047654045952688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,8192,5.047130584716797
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,65536,40.91031222873264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,5120,2.5041306813557944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,7168,4.525536855061849
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,6144,4.523773193359375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,4096,2.158271153767904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,6144,3.9436552259657116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,5120,3.4002925025092234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,3072,1.538439114888509
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,3584,2.4235750834147134
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,4096,2.8493440416124134
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,2560,1.7138764063517253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,3584,2.5524834526909723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,3072,2.296605428059896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,2048,1.261715571085612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,1024,0.6151742405361599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,1536,1.2824470731947157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,2048,1.570000860426161
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,2560,2.106581370035807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,1536,1.30367554558648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,768,0.5767404238382975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,512,0.3020862208472358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,256,0.19744355148739287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,128,0.18919377856784395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,1024,1.163560019599067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,64,0.18752977583143446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,768,1.0546666251288521
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,32,0.18951466348436144
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,512,0.9844346576266818
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,256,0.9026977750990125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,128,0.8000426822238498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,64,0.8057840135362414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,32,0.782857788933648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,12288,5.2857992384168835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,16384,7.065555572509766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,65536,31.09872097439236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,12288,7.015855153401692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,16384,9.277927822536892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,10240,4.4973805745442705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,8192,3.5141554938422312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,65536,35.694739447699654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,10240,5.902215321858724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,6144,2.647766325208876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,7168,4.223086039225261
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,8192,4.807112799750434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,7168,4.279961056179471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,5120,3.103110207451714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,4096,2.3876230451795792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,6144,3.8553678724500866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,5120,3.2428444756401906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,3584,2.036215040418837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,4096,2.6886649661593967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,3072,1.7439617580837672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,2560,1.4009084701538086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,2048,0.9272631539238824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,3072,2.1269573635525174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,3584,2.3910329606797958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,2560,1.8767422570122614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,1536,0.8805733256869845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,1024,0.5650586552090114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,768,0.4201351006825765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,512,0.29873598946465385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,2048,1.4465537601047094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,1536,1.203123516506619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,1024,0.9729813469780816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,256,0.21300800641377768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,128,0.17209777567121717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,768,0.9618399937947592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,64,0.1648728847503662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,32,0.18628977404700386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,512,0.8403155538770887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,128,0.6997813118828667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,256,0.7700871361626519
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,64,0.7141760190327963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,32,0.7158737712436252
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,12288,4.573841094970703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,65536,27.103968302408855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,12288,6.629884507921006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,16384,7.038329654269749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,10240,3.7617450290256076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,16384,8.672858344184027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,8192,3.0956009758843313
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,7168,2.631551954481337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,10240,5.649524264865452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,8192,4.538486904568143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,6144,2.8915430704752603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,65536,33.84292263454861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,7168,4.0501251220703125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,5120,2.588165283203125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,3584,1.7237422731187608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,4096,1.905990176730686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,6144,3.5394284990098743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,5120,3.0042525397406687
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,4096,2.6177804734971786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,3072,1.369590229458279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,2560,1.095671123928494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,3584,2.267117394341363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,2048,0.7423262066311307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,2560,1.6849564446343317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,3072,1.9948416815863714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,1536,0.7011235555013021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,1024,0.4072115686204698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,768,0.34888532426622176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,512,0.24483021100362143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,2048,1.2979502148098416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,256,0.14766132831573486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,1536,1.0843928654988606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,1024,0.8628791173299154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,128,0.1421955558988783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,768,0.8280488650004069
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,64,0.15329689449734157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,32,0.16176177395714655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,512,0.7517431047227648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,256,0.6689004368252225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,128,0.6529626846313477
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,64,0.6192257669236925
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,32,0.5806630982293023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,12288,4.241703033447266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,65536,23.43241034613715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,16384,5.09117677476671
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,12288,6.359499613444011
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,10240,3.2071573469373913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,8192,2.538069407145182
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,16384,8.335528055826822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,7168,2.2253617180718317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,10240,5.370864020453559
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,6144,2.627324422200521
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,8192,4.2543385823567705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,7168,3.8084504869249134
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,5120,2.2021990882025824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,4096,1.642571555243598
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,6144,3.327958212958442
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,65536,31.915408664279514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,5120,2.79292360941569
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,3072,1.0062977472941081
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,4096,2.4018898010253906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,3584,1.2825475268893771
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,2560,0.7491013209025065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,2048,0.6335760222540961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,3584,2.0501386854383683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,1536,0.5689359770880805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,1024,0.4006471104092068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,2048,1.1778524186876085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,3072,1.78784794277615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,512,0.19208533234066436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,768,0.37598133087158203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,1536,0.9621822569105359
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,2560,1.6022097269694011
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,256,0.12333422236972386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,128,0.12217422326405843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,1024,0.8057831128438314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,768,0.6896177927652994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,64,0.12131822109222412
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,512,0.620588461558024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,256,0.5832311312357584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,128,0.5390515327453613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,32,0.13642666074964735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,64,0.5256986618041992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,32,0.48602135976155597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,65536,19.731068929036457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,12288,3.043921788533529
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,16384,4.987120734320746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,10240,3.180094189114041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,12288,5.572349124484592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,8192,2.021376927693685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,16384,7.3510996500651045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,10240,4.679084353976779
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,7168,2.329958173963759
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,8192,3.857954661051432
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,6144,1.5339786741468642
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,5120,1.6023884879218206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,7168,3.3758665720621743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,4096,1.3111253314548068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,6144,2.889464908175998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,3584,0.9843368530273438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,5120,2.482596503363715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,3072,0.8443235291375054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,4096,1.9861297607421875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,2560,0.6408702002631294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,3584,1.7231458028157551
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,2048,0.5782693227132162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,1536,0.416136900583903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,3072,1.517123540242513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,2048,0.8512079980638293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,1024,0.30931467480129665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,65536,30.938802083333332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,2560,1.2526497311062283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,768,0.2323706679873996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,1536,0.6639431317647299
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,1024,0.44179378615485293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,256,0.09201866388320923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,128,0.06765777534908719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,512,0.16074666712019178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,768,0.42867112159729004
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,64,0.05946844153934055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,512,0.309641785091824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,256,0.2337395615047879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,128,0.2235004372066922
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,32,0.06572088930341932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,64,0.18532355626424155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,32,0.18744444847106934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,65536,15.882944742838541
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,12288,2.904893239339193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,10240,2.4906169043646917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,16384,4.372404310438368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,8192,1.8536150190565321
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,12288,5.377787695990668
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,16384,7.115667555067275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,7168,1.519057803683811
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,10240,4.55571026272244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,8192,3.6840659247504344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,6144,1.271235571967231
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,5120,1.0993768903944228
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,4096,0.9110533396402994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,7168,3.2990027533637156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,6144,2.787769741482205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,3584,0.7738915549384223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,5120,2.3350870344373917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,4096,1.8873405456542969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,3072,0.7580284542507596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,2560,0.6219546529981825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,3584,1.6484106911553278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,2048,0.5182266765170628
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,1536,0.3702550994025336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,3072,1.4214275148179796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,2048,0.818853325313992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,1024,0.2786666552225749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,2560,1.170273780822754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,1536,0.5889049106174046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,768,0.19671644104851616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,1024,0.39874932501051163
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,512,0.13482133547465006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,256,0.08082044124603271
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,128,0.04080355498525832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,768,0.3419013288285997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,512,0.29511022567749023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,64,0.046655111842685275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,256,0.21118222342597115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,32,0.029341333442264136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,128,0.17652532789442274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,64,0.16880888409084746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,32,0.15939644972483316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,65536,13.892703586154513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,65536,29.395833333333332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,16384,3.781617058648003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,12288,3.318017747667101
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,10240,2.2537449730767145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,8192,2.136757320827908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,16384,6.882319980197483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,12288,5.186750200059679
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,7168,1.3227208455403645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,8192,3.535074657864041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,10240,4.39597405327691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,6144,1.0304746627807617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,4096,0.7370862430996366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,5120,1.2301688724093967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,3584,0.8730035358005099
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,6144,2.6789972517225475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,7168,3.0986222161187063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,5120,2.243596394856771
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,3072,0.6075386471218532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,4096,1.8046018812391493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,2560,0.42965067757500547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,2048,0.40384356180826825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,65536,12.118815104166666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,1536,0.3422586652967665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,2048,0.743834654490153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,3584,1.5956604215833876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,3072,1.3435173034667969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,2560,1.1290790769788954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,1024,0.24520977338155112
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,768,0.18077066209581164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,256,0.07296533054775663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,512,0.11898222234514023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,1536,0.5264364348517524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,128,0.035841776265038386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,768,0.3081724378797743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,512,0.2633413208855523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,64,0.032292445500691734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,256,0.19717866844601104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,32,0.02659822172588772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,1024,0.37610401047600645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,64,0.14067110750410292
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,32,0.1441777812110053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,128,0.1602675517400106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,12288,2.3453155093722873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,65536,28.253406100802952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,16384,3.293419520060221
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,10240,1.5969911151462133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,8192,1.7706711027357314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,12288,5.112986670600043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,7168,1.5204106436835394
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,16384,6.724968804253472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,10240,4.415275573730469
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,8192,3.43486319647895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,6144,1.2891644371880424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,5120,0.828794691297743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,4096,0.8457591268751355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,7168,3.0276578267415366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,3584,0.7225404315524631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,6144,2.596639209323459
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,4096,1.7425102657741969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,5120,2.1766524844699435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,3072,0.6433626810709635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,3584,1.5112604565090602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,2560,0.4071057902442084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,2048,0.42681243684556747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,1536,0.27824799219767254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,1024,0.20308800538380942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,1536,0.49136712816026473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,3072,1.2852515114678276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,2048,0.6857422192891439
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,2560,1.0714835060967338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,768,0.14266044563717312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,512,0.10443911287519668
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,1024,0.36701689826117617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,256,0.06559733549753825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,512,0.22149599923027885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,128,0.03387022349569533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,768,0.2795075575510661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,256,0.15035822656419542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,64,0.0273964438173506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,128,0.14096089204152426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,32,0.021479111578729417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,64,0.12250310844845241
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,32,0.11946222517225479
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,12288,1.62574217054579
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,16384,2.0378523932562933
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,65536,14.647576226128471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,65536,27.362147013346355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,10240,1.7128355238172743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,8192,1.3819786707560222
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,16384,6.514248741997613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,7168,1.0944604873657227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,12288,4.928065829806857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,10240,4.285670386420356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,6144,0.9833484225802951
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,8192,3.3061608208550344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,5120,0.5724630885654026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,4096,0.4425937864515517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,65536,9.10408189561632
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,3584,0.4561920166015625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,3072,0.42652355300055605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,4096,1.6808924145168727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,6144,2.502908494737413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,7168,2.948633829752604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,5120,2.111138661702474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,2560,0.3280266655815972
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,2048,0.24631110827128092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,1536,0.20595110787285698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,3584,1.497231059604221
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,1024,0.14419733153449163
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,3072,1.2532089021470811
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,1536,0.43153778711954754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,768,0.11487555503845215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,2048,0.6438302463955349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,512,0.07995911439259847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,1024,0.2845582167307536
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,256,0.043170667356914945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,2560,1.0120568805270724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,128,0.027951111396153767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,512,0.21022933059268525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,768,0.2537279923756917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,64,0.02200711104604933
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,256,0.1328159968058268
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,128,0.11346756087409125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,32,0.0177875558535258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,64,0.0994337797164917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,32,0.09753688838746812
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,12288,1.1416390736897786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,16384,1.9852676391601562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,10240,1.0947511461046007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,8192,0.7774186664157443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,65536,26.794106377495662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,7168,0.6803244484795464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,12288,4.728097703721788
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,6144,0.5216062333848741
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,10240,3.956264919704861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,16384,6.254751841227214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,5120,0.49699820412529844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,8192,3.1971501244439016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,4096,0.391213337580363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,7168,2.7989955478244357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,65536,8.33282216389974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,3584,0.36546044879489475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,6144,2.408437305026584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,3072,0.3203066719902886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,2560,0.2150026692284478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,4096,1.5949652989705403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,2048,0.1918391121758355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,5120,1.9932496812608507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,1536,0.14859999550713435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,1024,0.10291733344395955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,3584,1.3866862191094294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,3072,1.1937964757283528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,2048,0.6005848778618706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,1024,0.2561475435892741
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,2560,0.9839501910739474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,768,0.07866844203737047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,1536,0.3922079934014215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,512,0.07039466831419203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,128,0.01984177695380317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,768,0.18796088960435656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,64,0.016672000288963318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,256,0.032660444577534996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,256,0.10672089126374985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,512,0.16132267316182455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,32,0.013819555441538492
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,128,0.08986400233374701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,64,0.08155022064844768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,32,0.07997777726915148
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,12288,0.7495164341396756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,16384,1.1261457867092557
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,10240,0.7148533397250705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,8192,0.5485431353251139
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,65536,4.237627665201823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,7168,0.4395288891262478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,65536,25.61440700954861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,12288,4.541560702853732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,10240,3.794225056966146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,8192,3.0559785630967884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,6144,0.3786453406016032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,5120,0.28255822923448354
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,4096,0.26041777928670246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,3584,0.22803023126390246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,16384,6.011311848958333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,3072,0.17896178033616808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,4096,1.5207227071126301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,7168,2.671744876437717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,6144,2.297642601860894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,5120,1.9236613379584417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,3584,1.3147431479560006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,2048,0.1284737719429864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,2560,0.15506933795081243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,1536,0.09158755673302545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,1024,0.06494577725728352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,2048,0.540703985426161
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,1536,0.3464035458034939
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,1024,0.2120008865992228
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,768,0.05576088693406847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,3072,1.1198595894707573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,512,0.03944089015324911
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,768,0.15347554948594835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,512,0.10382044315338135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,2560,0.9030026329888238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,128,0.014464888307783337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,256,0.022537777821222942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,256,0.07635644409391615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,64,0.011786667009194693
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,128,0.06444088617960612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,32,0.009973333113723332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,64,0.05890399879879422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,32,0.05687822235955132
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,12288,0.5722648832533095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,16384,0.7680195702446831
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,10240,0.5908711221483018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,8192,0.556854248046875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,65536,4.125482771131727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,7168,0.4611084196302626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,10240,3.713121626112196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,8192,2.9884950849745007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,6144,0.3392817709181044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,12288,4.439739651150174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,5120,0.3188764519161648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,4096,0.2604115539126926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,16384,5.894376966688369
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,65536,24.689108954535588
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,3584,0.21760800149705675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,3072,0.1828897794087728
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,5120,1.8682861328125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,7168,2.6191412607828775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,4096,1.4888916015625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,2560,0.15392177634769016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,6144,2.244991938273112
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,2048,0.11851733260684544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,3072,1.0873875088161893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,1536,0.08649333318074544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,1024,0.06699911091062757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,3584,1.3769173092312281
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,1536,0.3298737737867567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,768,0.044772442844178945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,2048,0.5211368666754829
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,1024,0.21618933147854277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,512,0.029603554142846003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,256,0.01793599956565433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,128,0.012169777519173093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,64,0.010030222435792288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,2560,0.8794310887654623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,512,0.08931199709574382
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,256,0.062478224436442055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,128,0.052080889542897545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,32,0.00849599970711602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,64,0.0469359987311893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,32,0.04523644513554043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,768,0.1665688885582818
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,16384,0.5332480006747776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,12288,0.4246542188856337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,10240,0.3635626633961995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,8192,0.2950346734788683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,65536,2.261120054456923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,7168,0.2482959959242079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,8192,2.9279467264811196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,10240,3.6438246832953562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,6144,0.19444355699751112
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,5120,0.15742222468058267
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,12288,4.353941175672743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,7168,2.5652088589138455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,4096,0.1261733373006185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,3584,0.10661422544055515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,16384,5.809406280517578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,3072,0.08632977803548177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,3584,1.257545789082845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,4096,1.4544258117675781
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,5120,1.8330987294514973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,2560,0.079894224802653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,65536,23.673160129123264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,6144,2.2039803398980036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,2048,0.05860622061623467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,1024,0.03432710965474447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,1536,0.047806223233540855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,768,0.027810666296217177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,1024,0.20055733786688912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,2048,0.49399201075236004
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,1536,0.3136168850792779
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,512,0.019863999552196927
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,256,0.012539555629094442
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,3072,1.057710223727756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,128,0.008453333543406593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,256,0.049823111957973905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,512,0.07307555278142293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,768,0.14594311184353298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,64,0.006784889019197888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,128,0.0399217771159278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,2560,0.8605822457207574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,32,0.006146666490369373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,64,0.03542222248183356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,32,0.03343733482890659
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,16384,0.5004702144198948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,12288,0.37117687861124676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,10240,0.310023996565077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,65536,1.6941164864434137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,8192,0.2528879907396105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,7168,0.19866489039527047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,10240,3.607865651448568
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,6144,0.1461955573823717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,8192,2.8944240146213107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,12288,4.318047841389974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,5120,0.0780622230635749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,7168,2.5228826734754777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,4096,0.06898044215308295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,3584,0.0644035538037618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,16384,5.7709867689344625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,3072,0.05051911208364698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,3584,1.2175786760118272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,4096,1.4056257671780055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,2560,0.03969866699642605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,2048,0.03141866789923774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,5120,1.7818097008599176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,6144,2.159168031480577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,65536,23.236402723524307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,1536,0.025591999292373657
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,3072,1.0272675620185006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,1024,0.01905955539809333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,768,0.015497777197096082
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,1024,0.17348978254530165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,512,0.011120888921949597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,2048,0.4908542103237576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,256,0.0074444446298811175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,1536,0.3018791145748562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,128,0.005505777895450592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,64,0.004679111142953237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,512,0.0529093345006307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,768,0.11646933025783962
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,256,0.035903112755881414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,32,0.004074666649103165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,128,0.027496889233589172
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,64,0.023564444647894964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,2560,0.8403857549031576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,32,0.022172444396548804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,16384,0.440887106789483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,12288,0.34396976894802517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,10240,0.2662631140814887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,8192,0.21912533707088896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,65536,1.6488907072279189
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,7168,0.21166843838161895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,8192,2.864173253377279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,6144,0.10433689090940688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,10240,3.5826924641927085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,5120,0.04455111092991299
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,7168,2.4943065643310547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,4096,0.04205866654713949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,12288,4.294850667317708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,3584,0.04115733173158433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,16384,5.752152760823567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,3072,0.034733331865734525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,4096,1.381954616970486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,3584,1.1966266632080078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,6144,2.130041758219401
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,2560,0.029105779197480943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,2048,0.02181511123975118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,5120,1.7609991497463648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,1536,0.018647111124462552
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,1024,0.012753778033786349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,1024,0.12437244256337483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,2048,0.468689759572347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,768,0.010583110981517367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,1536,0.26123200522528756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,512,0.008358221915033128
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,256,0.0057857777509424425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,768,0.0694942209455702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,512,0.04596889019012451
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,2560,0.8220826784769694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,256,0.030801776382658217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,128,0.004683555414279302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,32,0.0036480000449551474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,64,0.003993777765168084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,3072,1.0113591088189018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,128,0.023079110516442194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,64,0.019332443674405415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,32,0.017484444710943434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,65536,22.925799899631077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,16384,0.4348319901360406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,12288,0.34264087677001953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,10240,0.2586391237046984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,8192,0.21313956048753527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,65536,1.629545741611057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,7168,0.2010791169272529
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,8192,2.86045773824056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,10240,3.5847617255316844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,6144,0.08497333526611328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,12288,4.405577765570746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,5120,0.04481777879926893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,4096,0.031566222508748375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,16384,5.729053497314453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,3584,0.03242933419015672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,4096,1.3861804538302953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,3072,0.027687110834651526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,5120,1.7661795086330836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,7168,2.492955525716146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,6144,2.1347334120008683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,2560,0.02345866627163357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,2048,0.016893333858913846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,1536,0.015327110886573792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,1536,0.2685813373989529
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,3584,1.2003466288248699
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,2048,0.46378575430976016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,1024,0.010097777677906884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,768,0.008759111166000366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,512,0.005965333431959152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,768,0.06924533181720309
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,1024,0.11918577882978652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,65536,22.649156358506943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,256,0.004563555535342959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,512,0.04551644457711113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,3072,1.0082409116956923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,2560,0.8183733092414008
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,256,0.030402666992611353
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,64,0.0032435556252797446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,128,0.02307022280163235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,32,0.002942222274012036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,128,0.003986666599909465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,64,0.019387554791238572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,32,0.017363554901546903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,12288,0.33728888299730087
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,16384,0.4075022273593479
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,10240,0.2557857831319173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,8192,0.2082186672422621
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,65536,1.6301662656995985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,7168,0.18573244412740073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,8192,2.8566737704806857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,6144,0.06299466556972928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,10240,3.5839254591200085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,5120,0.03733333283000522
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,7168,2.497555626763238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,12288,4.291856977674696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,4096,0.028203556934992473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,3584,0.02747111188040839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,3584,1.1949599583943684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,16384,5.710253397623698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,3072,0.02345866627163357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,4096,1.3887928856743708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,2560,0.020399999287393358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,5120,1.7673821979098852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,2048,0.015091554986106025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,1536,0.012977778083748288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,6144,2.1355857849121094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,1024,0.009479111267460717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,1536,0.26851288477579754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,768,0.007742221984598372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,1024,0.11868445078531902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,2048,0.4696897930569119
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,512,0.005666666560702854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,768,0.07202311356862386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,256,0.004567111117972268
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,128,0.0034213333080212274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,512,0.045347554816140064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,2560,0.8158515294392904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,256,0.030312889152103003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,128,0.022967999180157978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,64,0.0027626665929953256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,32,0.0026133333643277488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,3072,1.009014235602485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,32,0.017362667454613578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,64,0.019320888651741874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,65536,22.814080132378475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,65536,22.707539876302082
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,10240,11.903718736436632
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,12288,13.640510559082031
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,65536,22.610024346245662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,16384,18.084042019314236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,10240,20.126578436957466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,8192,9.419081793891058
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,12288,24.244260999891495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,8192,16.736797756618923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,7168,8.948499891493055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,6144,7.2589221530490455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,7168,14.05096689860026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,4096,5.027313656277127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,6144,12.021032545301649
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,5120,6.523735894097222
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,16384,32.753014458550346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,4096,7.972162034776475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,5120,10.224468655056423
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,3584,4.518375820583768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,3072,4.006969028049045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,3584,7.016964382595486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,2560,5.009534200032552
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,1536,3.034192827012804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,2048,3.2586390177408853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,3072,6.049820370144314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,2560,3.564644283718533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,2048,4.064217885335286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,1536,2.6736026340060763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,512,1.4684844546847875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,1024,3.4440434773763022
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,768,2.4130942026774087
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,1024,2.0910720825195312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,768,1.9533297220865886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,256,1.0352213117811415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,512,1.792710198296441
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,64,1.0774737464057076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,128,1.095589319864909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,256,1.544806268480089
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,32,1.2024186452229817
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,64,1.3045057720608182
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,128,1.4587910970052083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,32,1.3451422585381403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,10240,4.204416063096788
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,10240,5.0338694254557295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,12288,5.03689702351888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,12288,6.073855929904514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,16384,6.597535875108506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,16384,8.153615315755209
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,8192,3.2732649909125437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,8192,4.164687262641059
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,7168,3.5957840813530813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,7168,3.0758416917588978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,6144,3.1215485466851125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,6144,2.6515519883897567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,4096,1.6294640435112848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,5120,2.211947547064887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,4096,2.0757901933458114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,5120,2.5493272145589194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,3584,1.7322301864624023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,3584,1.474040879143609
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,2048,0.9874159495035807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,3072,1.267441749572754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,3072,1.443842675950792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,2560,1.0848382314046223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,2560,1.4213093651665583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,1536,0.774578677283393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,768,0.5282328923543295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,1024,0.6738817956712512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,2048,0.9214898215399848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,1024,0.5970311164855957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,256,0.231041775809394
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,1536,0.767265796661377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,512,0.3976408905453152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,768,0.5599795447455512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,128,0.19593689176771376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,256,0.38802311155531144
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,512,0.5301875538296169
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,32,0.18851733207702637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,64,0.20180622736612955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,128,0.38495643933614093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,64,0.34920265939500594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,32,0.33704710006713867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,12288,4.2705535888671875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,12288,5.069586859809028
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,16384,5.629905700683594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,16384,6.2176115247938375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,10240,3.8331565856933594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,8192,3.0626568264431424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,10240,3.6175698174370656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,8192,2.9027279747856984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,7168,2.629664103190104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,7168,2.4854674869113498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,6144,2.796703126695421
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,6144,2.2082231309678817
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,65536,26.4503411187066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,5120,1.8501226637098525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,3584,1.2365101708306205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,4096,1.3539910846286352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,4096,1.472373326619466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,5120,1.8353084988064234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,3072,1.06095462375217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,2560,1.0882292853461373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,3072,1.0132008658515084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,3584,1.1637840270996094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,2560,0.8646551238165961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,1536,0.5302169058057997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,2048,0.7127288712395562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,2048,0.7584151162041558
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,1024,0.35364709960089785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,1536,0.5849857860141331
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,768,0.3480844497680664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,512,0.24842400021023223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,1024,0.5456159909566244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,768,0.4398613240983751
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,256,0.16982844140794542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,128,0.14486932754516602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,512,0.3573937680986192
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,65536,31.180460611979168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,64,0.15035288863711888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,256,0.3350817892286513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,32,0.15097510814666748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,128,0.29226933585272896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,64,0.26711644066704643
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,32,0.27140267690022785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,12288,3.857122633192274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,16384,5.08001708984375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,16384,5.1815672980414496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,65536,22.384840223524307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,12288,3.944053226047092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,65536,23.474000718858505
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,10240,3.2044525146484375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,8192,2.5845457712809243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,10240,3.3074107699924045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,8192,2.595200856526693
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,7168,2.330937703450521
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,7168,2.6062694125705295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,6144,1.9051645067003038
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,5120,1.5968169106377497
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,6144,1.9490036434597438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,4096,1.2636924319797092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,3584,1.0868764453464084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,5120,1.648558192782932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,4096,1.2592426935831706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,3584,1.0506888495551217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,3072,1.2621208826700847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,3072,0.8996889326307508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,2560,0.780802673763699
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,2048,0.6093689070807563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,2560,0.7503662109375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,1024,0.3518586688571506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,1536,0.5758328967624241
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,768,0.2434337668948703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,2048,0.6212115817599827
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,1536,0.5168435308668349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,1024,0.44150577651129824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,512,0.22712000211079916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,256,0.12824088997311062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,512,0.3209759924146864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,64,0.11560800340440537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,256,0.26592445373535156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,128,0.12021777364942764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,768,0.3742382261488173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,32,0.11538399590386285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,128,0.2688613202836779
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,64,0.23001954290601942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,32,0.22369244363572863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,12288,3.0938826666937937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,16384,4.073919084337023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,12288,3.8150126139322915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,16384,5.021172417534722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,10240,2.5748657650417752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,8192,2.211398230658637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,65536,19.682164510091145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,10240,3.2368719312879772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,7168,1.7847084469265406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,65536,20.594677395290798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,8192,2.618920008341471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,6144,1.5869323942396376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,7168,2.316889868842231
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,5120,1.189125378926595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,4096,1.0830515755547416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,4096,1.2664808697170682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,6144,2.0184648301866317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,3584,0.8969732920328776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,5120,1.7865653567843969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,3072,0.8331857787238227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,3584,1.1548675960964627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,2560,0.7157768673366971
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,2048,0.5921529134114584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,1536,0.3563466601901584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,2560,0.8692444695366753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,2048,0.7815244462754992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,1024,0.33506756358676487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,3072,1.0042639838324654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,768,0.23851201269361708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,512,0.16603733433617487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,256,0.09487289190292358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,128,0.05950488646825155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,1536,0.6814719835917155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,1024,0.5965306493971083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,64,0.07606489128536649
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,768,0.540063116285536
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,32,0.05492888887723287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,512,0.4886168903774685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,256,0.4650728967454698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,128,0.42067911889817977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,64,0.42090845108032227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,32,0.40692978435092503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,12288,2.739295111762153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,16384,3.5876791212293835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,65536,15.789706759982638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,12288,3.6064410739474826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,16384,4.728552924262153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,10240,2.292994605170356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,8192,1.8407334221733942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,65536,18.357004801432293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,10240,3.070027457343208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,7168,1.827297846476237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,8192,2.4535715315077042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,6144,1.3248151143391926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,7168,2.1805903116861978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,4096,0.8351848920186361
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,5120,1.2647902170817058
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,4096,1.2294026480780709
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,3584,0.8773982259962293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,5120,1.6102515326605902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,6144,1.9090071784125433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,3072,0.6108826531304253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,2560,0.599547545115153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,3584,1.0850452846950955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,3072,0.9535129335191516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,2048,0.444717354244656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,1536,0.39368176460266113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,2560,0.8020053439670138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,1024,0.25665511025322807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,2048,0.6980373594495984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,768,0.2125422159830729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,512,0.13818666670057508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,1536,0.6166657871670193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,1024,0.5459155506557888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,256,0.080022222465939
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,768,0.481112003326416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,128,0.0483511123392317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,64,0.06799288590749104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,32,0.03546844588385688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,512,0.4453973240322537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,128,0.3673120074801975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,256,0.40271110004848903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,64,0.37125688129001194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,32,0.34067289034525555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,12288,2.374696943495009
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,65536,13.942177666558159
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,16384,3.2047316233317056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,12288,3.3944791158040366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,10240,2.0789243910047746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,16384,4.466563754611545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,8192,1.5559217664930556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,7168,1.3460729387071397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,6144,1.1495119730631511
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,10240,2.937849680582682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,8192,2.2707973056369357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,65536,17.457870483398438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,7168,2.005992889404297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,5120,0.928534189860026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,4096,1.059766239590115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,3584,0.7789768642849393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,5120,1.4840773476494684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,6144,1.7599591149224176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,4096,1.0908435185750325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,3584,0.9428062438964844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,3072,0.6721857918633355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,2560,0.5180515713161892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,2048,0.41685954729715985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,3072,0.8749022483825684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,1536,0.32733154296875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,2560,0.715120898352729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,1024,0.26319021648830837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,2048,0.610589345296224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,768,0.1910897758271959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,1536,0.5544631216261122
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,256,0.08060710959964328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,512,0.133861329820421
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,128,0.05291822221544054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,64,0.037138667371537946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,1024,0.46903377109103733
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,32,0.031585776143603854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,768,0.4314124319288466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,512,0.3814817799462213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,256,0.33565688133239746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,128,0.3080764346652561
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,64,0.3024711079067654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,32,0.31265245543585884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,12288,2.09044435289171
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,16384,2.6340914832221136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,12288,3.177656809488932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,10240,1.595456017388238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,16384,4.179300520155165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,65536,14.663046942816841
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,8192,1.3427609337700739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,7168,1.1441564559936523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,10240,2.6737431420220266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,6144,0.9657600190904406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,8192,2.119751188490126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,5120,0.7838906712002225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,7168,1.9807012346055772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,4096,0.6818488968743218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,6144,1.6394507090250652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,65536,16.476665920681423
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,3584,0.6055119832356771
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,5120,1.392913818359375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,4096,0.9667812983194987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,3072,0.5197866757710775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,3584,0.8571137852138944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,2560,0.4495075543721517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,2048,0.30673689312405056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,3072,0.7412880261739095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,1536,0.27908534473843044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,2560,0.6184924443562826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,1024,0.15913955370585123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,768,0.1461662186516656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,2048,0.5579066806369358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,512,0.10563021898269653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,256,0.0704524450831943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,1536,0.5028933419121636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,128,0.039084444443384804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,512,0.3219031227959527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,768,0.3736284573872884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,1024,0.41054754787021214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,64,0.02867199977238973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,256,0.2681777742173937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,32,0.0340648889541626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,128,0.2672293451097277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,32,0.24172533882988823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,64,0.26015644603305393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,12288,1.5603698094685872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,16384,2.244052463107639
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,65536,11.700153774685331
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,8192,1.0135670767890084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,10240,1.2313867145114474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,16384,3.7606463962131076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,12288,2.8286719852023654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,7168,1.1469075944688585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,10240,2.444697697957357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,8192,1.8979199727376301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,6144,0.8421182102627225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,5120,0.6139831013149685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,7168,1.642977820502387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,4096,0.5345795419481065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,6144,1.4253528383043077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,3584,0.47182665930853945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,4096,0.7697955767313639
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,65536,15.560219658745659
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,3072,0.4048613442314996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,5120,1.1509724722968209
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,2560,0.29603645536634654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,3584,0.6524568663703071
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,3072,0.5382933086819118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,2048,0.25928089353773326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,1536,0.18920977910359701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,1024,0.1382311052746243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,2048,0.3416764471266005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,768,0.10458933644824558
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,1536,0.29967912038167316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,2560,0.4323759873708089
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,512,0.07774400048785739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,1024,0.2353280120425754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,256,0.04697511262363858
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,768,0.18954133987426758
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,128,0.02453600035773383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,512,0.15374667114681667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,64,0.022319111559126113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,32,0.017345777816242643
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,128,0.1149937841627333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,256,0.12232266532050239
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,64,0.09971288839975993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,32,0.09977155261569554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,65536,9.41090816921658
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,12288,1.3915049235026042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,16384,1.9328933291965062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,10240,1.430890719095866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,12288,2.734640969170464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,8192,0.9867395824856229
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,7168,0.7810871336195203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,16384,3.6439946492513022
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,10240,2.275949266221788
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,5120,0.5988737742106119
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,6144,0.7415493329366049
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,8192,1.836465835571289
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,7168,1.5988586213853624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,4096,0.4847031169467502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,3584,0.46941423416137695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,4096,0.7226079834832085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,6144,1.3617199791802301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,5120,1.1112026638454862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,3584,0.6131919754876031
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,3072,0.4029120074378119
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,2560,0.3200017876095242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,3072,0.4840390947129991
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,2048,0.28078222274780273
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,1536,0.22728443145751953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,2560,0.38799111048380536
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,2048,0.3423057662116156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,768,0.09974933332867092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,1536,0.2736462222205268
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,512,0.07326489024692111
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,1024,0.13373777601453993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,1024,0.22319287723965117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,768,0.18019378185272217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,256,0.036736889017952815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,128,0.023344889283180237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,64,0.020184889435768127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,512,0.13369777467515734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,32,0.016216889023780823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,256,0.11088355382283528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,128,0.09967466857698228
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,64,0.09240621990627712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,32,0.08636889192793105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,65536,7.546434190538194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,65536,14.923680623372396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,12288,1.2721182505289714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,16384,1.5812693701850042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,10240,0.9009066687689887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,8192,0.7771457566155328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,12288,2.6421385871039496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,10240,2.2226640913221574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,16384,3.5322435167100696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,7168,0.648246235317654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,6144,0.5522141986423069
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,8192,1.7633946736653645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,5120,0.45296091503567165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,4096,0.33012355698479545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,3584,0.3239155610402425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,7168,1.5508675045437281
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,5120,1.069359991285536
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,4096,0.7112559742397733
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,6144,1.3728151321411133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,3072,0.25475022527906627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,2560,0.23619998825920951
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,2048,0.19092710812886557
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,3584,0.5752497778998481
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,65536,6.133324517144097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,2560,0.3635439872741699
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,1536,0.15227733718024358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,3072,0.4497804641723633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,768,0.07985333601633708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,1024,0.11457955837249756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,2048,0.2961680094401042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,512,0.06830755869547527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,256,0.03331111205948724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,1024,0.18082311418321398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,1536,0.22242755360073516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,128,0.021884444687101576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,768,0.1445831060409546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,512,0.13075910674201116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,64,0.01867288847764333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,32,0.014005333185195923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,256,0.10146044360266791
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,128,0.08676711055967544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,64,0.07762933439678617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,32,0.08112533224953546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,12288,0.9104435178968641
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,65536,14.381378173828125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,16384,1.5296932856241863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,10240,0.7761537763807508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,8192,0.7490684191385905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,7168,0.6887075636121961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,12288,2.545809851752387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,10240,2.144368913438585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,16384,3.3758665720621743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,8192,1.6924141777886286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,6144,0.561896006266276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,5120,0.44988266626993817
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,4096,0.37014932102627224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,7168,1.482288890414768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,3584,0.3477715651194255
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,65536,4.870831807454427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,4096,0.6421475410461426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,5120,1.0171368916829426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,6144,1.2465964423285592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,3072,0.30480533175998265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,2560,0.2513679928249783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,3584,0.5394088957044814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,2048,0.2029973268508911
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,1536,0.15720444255405003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,2048,0.27812356419033474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,1024,0.08805244498782688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,2560,0.33490933312310117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,3072,0.4510640038384332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,768,0.07695911327997844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,512,0.049023999108208545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,1024,0.1566737757788764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,256,0.02788089050187005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,768,0.13345332940419516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,128,0.016872889465755887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,64,0.015517334143320719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,512,0.10305511289172703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,32,0.012659555508030785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,128,0.07784444755978055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,1536,0.20722666051652697
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,64,0.06872177786297269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,256,0.08295910888248019
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,32,0.0680168867111206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,12288,0.7383297814263238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,16384,1.0600026448567708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,10240,0.6650071144104004
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,65536,13.937958611382378
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,8192,0.5222684542338053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,12288,2.4532409244113498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,7168,0.4042239983876546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,6144,0.3800240092807346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,10240,2.058834711710612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,16384,3.2738622029622397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,8192,1.6656488842434352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,5120,0.33393332693311906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,65536,4.090172237820096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,4096,0.24537687831454805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,3584,0.23847911092970106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,7168,1.418735080295139
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,4096,0.6090533468458388
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,3072,0.19877066877153185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,6144,1.2062604692247179
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,5120,0.9795849058363172
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,3584,0.5082551108466254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,2560,0.14461510711246067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,2048,0.1277208858066135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,1536,0.09514133135477702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,2560,0.3171982235378689
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,1024,0.07214044200049506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,2048,0.2418444421556261
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,3072,0.3926071061028375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,768,0.0595102243953281
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,1536,0.17632710933685303
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,512,0.03798222210672166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,256,0.021497777766651575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,768,0.10974133014678955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,512,0.08532710870107015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,128,0.014320888453059726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,1024,0.14002845022413465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,64,0.011991110940774282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,256,0.06939999924765693
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,32,0.010068444742096795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,128,0.061807109249962695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,64,0.05604444609747993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,32,0.05607822206285265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,12288,0.61044798956977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,16384,0.8654568990071615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,10240,0.5928311347961426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,8192,0.4333777692582872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,65536,13.377902560763888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,7168,0.396945767932468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,12288,2.3710685306125217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,65536,3.486744986640083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,16384,3.148900349934896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,6144,0.3558808962504069
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,10240,1.9777200486924913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,5120,0.31000622113545734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,4096,0.22158311472998726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,8192,1.5635662078857422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,3584,0.19551911618974474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,3072,0.18652355670928955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,7168,1.3644675148857965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,4096,0.5823333528306749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,6144,1.161603503757053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,5120,0.9411804411146375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,2560,0.14487377802530924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,3584,0.47743468814426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,2048,0.10848533444934422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,1536,0.08676444159613715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,1024,0.055952886740366616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,2560,0.2866364320119222
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,3072,0.3656773302290175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,2048,0.23851288689507377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,512,0.029262221521801416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,1024,0.10346755716535781
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,256,0.017874666386180453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,1536,0.1788008875317044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,768,0.04267200165324741
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,768,0.08185066779454549
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,128,0.012073777616024017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,512,0.06871822145250109
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,32,0.008325333396593729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,64,0.010335110955768162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,256,0.05729689200719198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,128,0.049363556835386485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,64,0.045159111420313515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,32,0.04408266809251574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,12288,0.3846728801727295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,16384,0.5186008877224392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,10240,0.33269421259562176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,8192,0.290736887190077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,65536,2.3175830841064453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,12288,2.2691004011366105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,65536,12.830516391330294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,10240,1.8952159881591797
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,7168,0.2429413265652127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,5120,0.16657243834601507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,6144,0.18786133660210502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,8192,1.4977910783555772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,4096,0.12484799491034614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,3584,0.10814755492740208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,16384,3.01294305589464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,3072,0.08774666653739081
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,4096,0.5404551294114854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,2560,0.07719822062386407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,6144,1.096017731560601
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,3584,0.4424533314175076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,5120,0.9032861921522352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,7168,1.2994178136189778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,2048,0.06464266777038574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,1536,0.047205332252714366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,1024,0.03637866510285272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,3072,0.3331342273288303
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,768,0.029145777225494385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,2560,0.2571173244052463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,2048,0.20752622021569145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,512,0.020097777247428894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,256,0.012799999780125089
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,1024,0.08263555500242445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,1536,0.14707911014556885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,768,0.06601066721810235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,128,0.00832088871134652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,512,0.054089777999454074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,64,0.006783999916579988
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,256,0.04317155480384827
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,32,0.005968889014588461
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,128,0.03679822219742669
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,64,0.03353955679469638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,32,0.032487998406092324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,12288,0.29100267092386883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,16384,0.40435200267367893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,10240,0.2683448791503906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,65536,2.148024876912435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,8192,0.28518133693271214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,7168,0.2164755529827542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,12288,2.2735093434651694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,10240,1.8918836381700304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,65536,12.344698588053385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,6144,0.16920799679226348
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,16384,3.0134889814588757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,5120,0.1606986655129327
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,4096,0.1175511148240831
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,3584,0.10487822029325698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,5120,0.8921244939168295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,8192,1.5020497639973958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,4096,0.5563928816053603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,6144,1.0993209415011935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,3072,0.10399288601345485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,7168,1.2983555263943143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,2560,0.08981688817342122
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,3584,0.4433422353532579
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,2048,0.07687199778027005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,1536,0.0511448880036672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,1024,0.03443911009364658
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,768,0.027982221709357366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,2560,0.2534257835812039
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,3072,0.33653778500027126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,1536,0.13902666833665636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,1024,0.07699288924535115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,512,0.019523556033770244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,2048,0.20366222328609893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,256,0.011893333660231696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,128,0.00811111099190182
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,768,0.060363557603624135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,64,0.00620977787507905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,512,0.04993955625428093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,256,0.03937422235806783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,32,0.00536088893810908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,128,0.03363644414477878
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,64,0.030774222479926214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,32,0.02956177790959676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,12288,0.20458577738867867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,16384,0.2783884472317166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,10240,0.1597599983215332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,8192,0.1381440030203925
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,65536,1.3662275738186305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,7168,0.1586302253935072
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,8192,1.4330000347561305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,10240,1.8175368838840062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,6144,0.0896186696158515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,12288,2.1913511488172746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,5120,0.07710044251547919
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,4096,0.07106044557359484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,7168,1.2429706785413954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,3584,0.054690665668911405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,65536,11.831533643934462
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,16384,2.904019461737739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,4096,0.5039422247144911
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,6144,1.0539573033650715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,3072,0.04742666747834948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,2560,0.040989332728915744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,5120,0.8514657550387912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,3584,0.4125022093454997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,2048,0.03916622201601664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,1536,0.026691555976867676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,3072,0.3045395480261909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,1024,0.01884622209601932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,2048,0.1797057787577311
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,768,0.015477332803938123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,2560,0.23037778006659615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,1024,0.05752799908320109
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,1536,0.11414755715264215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,512,0.01110666659143236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,256,0.007460444337791867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,128,0.005508444375462002
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,64,0.004595555365085602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,512,0.037816888756222196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,768,0.04735022121005588
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,32,0.003948444293604957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,256,0.028883556524912517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,128,0.024326221810446844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,64,0.021464889248212177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,32,0.020615999897321064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,16384,0.2485662301381429
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,12288,0.19047289424472383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,10240,0.0874942209985521
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,8192,0.07322666380140516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,65536,0.8859199947781033
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,7168,0.062353776560889355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,8192,1.4084658092922633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,10240,1.7919315761990016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,6144,0.05099111133151584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,5120,0.0443742209010654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,7168,1.2216711044311523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,12288,2.15739991929796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,4096,0.0365608897474077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,16384,2.887064827813042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,4096,0.481935977935791
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,3584,0.03312355611059401
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,5120,0.8369102478027344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,3072,0.02966844373279148
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,6144,1.0321448644002278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,2560,0.028174221515655518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,2048,0.022127999199761286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,65536,11.854285346137154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,2560,0.19065599971347383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,2048,0.13011911180284289
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,3584,0.3885928789774577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,1536,0.018408000469207764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,1024,0.012649777862760754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,3072,0.27429244253370494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,768,0.01069866700304879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,512,0.008169777691364288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,1536,0.07814400063620673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,256,0.005948444621430502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,768,0.04094666573736403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,1024,0.04939200149642097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,128,0.004646222210592694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,64,0.003670222229427762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,512,0.03302755620744493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,256,0.02473955518669552
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,32,0.003592888928121991
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,128,0.020076443751653034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,64,0.017627555463049147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,32,0.016756445169448853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,12288,0.13036978244781494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,16384,0.22608444425794813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,10240,0.044340444935692676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,8192,0.03472888800832961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,65536,0.8512106471591525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,7168,0.03147911032040914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,8192,1.4068347083197699
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,10240,1.779776891072591
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,6144,0.027823110421498615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,5120,0.024275556206703186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,7168,1.211652437845866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,4096,0.0247999992635515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,12288,2.147957271999783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,3584,0.023603555228975084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,16384,2.8680462307400174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,4096,0.4769262207878961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,3072,0.021353777911927965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,3584,0.3771831194559733
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,2560,0.01957066688272688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,5120,0.8323218027750651
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,2048,0.015307555596033732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,1536,0.012686221963829465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,6144,1.0246844821506076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,3072,0.24464178085327148
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,1024,0.009000889129108852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,2560,0.1709208885828654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,2048,0.10226666927337646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,768,0.0074924445814556545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,65536,11.43292236328125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,512,0.005945777727497949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,1024,0.04708622230423821
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,1536,0.06677599747975667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,512,0.03072533342573378
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,768,0.0391244457827674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,128,0.003687111039956411
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,256,0.004505777938498391
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,32,0.0028684443483750024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,64,0.003167111012670729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,256,0.022553778356975977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,64,0.015858666764365304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,128,0.018231999542978074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,32,0.014927110738224454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,12288,0.08484711249669392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,16384,0.22071733739640978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,10240,0.0403039985232883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,8192,0.033167110549079046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,65536,0.8302053345574273
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,7168,0.04886933167775472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,8192,1.405662218729655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,6144,0.04178399840990702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,10240,1.7882436116536458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,7168,1.211690690782335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,5120,0.0361768901348114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,12288,2.1438231998019748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,4096,0.027609777119424608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,3584,0.026213333010673523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,3584,0.3702079984876845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,4096,0.47133954366048175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,3072,0.02237333357334137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,16384,2.8708826700846353
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,2560,0.019621334142155118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,6144,1.0196026696099174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,2048,0.015174221661355762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,1536,0.01256444470749961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,5120,0.8267297744750977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,3072,0.2485128773583306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,2048,0.10265422529644436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,1024,0.008781332936551835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,1536,0.06572266419728597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,768,0.007407111426194509
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,2560,0.16725777255164254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,512,0.005642666584915585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,1024,0.04692088895373874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,65536,11.390408833821615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,768,0.03881511092185974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,256,0.0041422223051389056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,64,0.002504000026318762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,512,0.030449777841567993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,256,0.022272000710169475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,128,0.0034231110993358824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,32,0.002393777792652448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,128,0.018234666850831773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,64,0.015754666593339708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,16384,0.21567822827233207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,32,0.014838222000333997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,12288,0.06140355269114176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,10240,0.03916800022125244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,65536,0.8190808826022677
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,8192,0.032242665688196816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,7168,0.04066310988532172
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,10240,1.7876781887478297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,8192,1.3984631432427301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,6144,0.032617777585983276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,5120,0.02832533253563775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,12288,2.150096893310547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,4096,0.021083555287784998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,16384,2.8741529252794056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,3584,0.021677333447668288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,7168,1.210096041361491
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,4096,0.4759350882636176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,3072,0.01811822255452474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,2560,0.015081778168678284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,5120,0.8303635385301379
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,6144,1.021084467569987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,2048,0.011563555234008364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,3584,0.36535734600490993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,1536,0.009958222508430481
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,1024,0.007189333438873291
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,2560,0.16501243909200033
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,3072,0.26063821050855845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,2048,0.10118755367067124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,768,0.00591555568906996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,65536,11.385700649685331
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,1536,0.06474400228924222
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,1024,0.04687111245261299
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,512,0.0046568889584806235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,128,0.0028533333291610083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,512,0.030127998855378892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,256,0.003590222034189436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,768,0.03858577873971727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,64,0.002431111203299628
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,128,0.018111111389266122
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,256,0.02218933403491974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,32,0.002373333399494489
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,64,0.01571999986966451
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,32,0.014808000789748298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,65536,11.360872904459635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,10240,5.726212395562066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,12288,7.005255805121528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,65536,11.361158582899305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,16384,8.970373365614149
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,10240,10.380763583713108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,8192,4.555033789740668
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,7168,4.136673821343316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,8192,8.138242933485243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,12288,12.274828592936197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,7168,7.049654218885634
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,6144,3.514322704739041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,5120,2.976348453097873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,6144,6.0623902214898004
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,4096,2.487229241265191
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,5120,5.010231018066406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,4096,4.023647308349609
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,16384,16.383553398980034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,3584,2.2389058007134333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,3072,2.149743186102973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,2560,2.5536443922254777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,3584,3.547032038370768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,2560,1.7285644743177626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,2048,2.0506568484836154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,2048,1.529891543918186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,3072,3.0652258131239147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,1536,1.519768926832411
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,512,0.6625368860032823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,1536,1.2903066211276584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,1024,1.1457564036051433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,256,0.5377697944641113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,768,0.9513342115614148
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,768,1.0243270662095811
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,512,0.8906400468614367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,1024,1.0717324150933158
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,64,0.48155466715494794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,128,0.49740001890394425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,32,0.601602660285102
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,64,0.7114986843532987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,32,0.66921509636773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,128,0.7526355319552951
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,256,0.8071626557244195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,12288,2.4666773478190103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,10240,2.5956266191270614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,12288,3.0679334004720054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,16384,3.187316470675998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,16384,4.1792958577473955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,8192,1.455916404724121
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,8192,2.0598231421576605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,7168,1.4277555677625868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,10240,1.995055940416124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,7168,1.7676702075534398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,6144,1.5215750800238714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,6144,1.0767528745863173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,4096,0.9732533560858833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,5120,0.9645955827501086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,5120,1.2710960176255968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,4096,0.7624337938096789
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,3584,0.8848409122890897
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,3072,0.6350337664286295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,3584,0.6809671190049914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,2560,0.6417822308010525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,2560,0.5295964346991645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,3072,0.725671132405599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,2048,0.5173528989156088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,1536,0.3765679995218913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,1024,0.28194578488667804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,1024,0.35559288660685223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,2048,0.45910045835706925
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,512,0.17748799588945177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,768,0.30968978669908315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,768,0.22811465793185762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,128,0.05585777759552002
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,1536,0.40221688482496476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,256,0.09430222378836738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,64,0.04354044463899401
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,512,0.26031290160285103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,256,0.21375111738840738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,32,0.031439112292395696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,128,0.20537422762976754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,32,0.18395288785298666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,64,0.1819617748260498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,12288,2.020653406778971
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,12288,2.2880062527126737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,16384,2.7723812527126737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,16384,3.090609868367513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,10240,1.926083458794488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,10240,1.7343351576063368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,8192,1.1824951171875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,8192,1.5346817440456813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,65536,12.944789462619356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,7168,1.0365236070421007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,6144,0.9017990960015191
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,6144,1.092942237854004
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,7168,1.3109377755059135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,5120,1.0999395582411025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,4096,0.7678382131788465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,3584,0.6746968693203397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,5120,0.7605031331380209
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,4096,0.6235057512919108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,3072,0.5766791237725152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,3584,0.6044026480780708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,2560,0.4607226583692763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,2560,0.43131645520528156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,3072,0.5006053182813857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,65536,15.9383053249783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,1536,0.3045244481828478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,2048,0.37642576959398055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,1536,0.3065653377109104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,1024,0.1979102161195543
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,768,0.16332622369130453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,2048,0.37224798732333714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,1024,0.27287289831373424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,256,0.0701999995443556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,768,0.22094666957855225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,64,0.03259999884499444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,512,0.19011733267042372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,256,0.16258133782280818
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,128,0.16228977839152017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,128,0.04438666502634684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,32,0.025648888614442613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,512,0.13654222753312853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,64,0.13705778121948242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,32,0.13943999343448216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,12288,2.115687052408854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,12288,1.9061652289496527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,16384,2.5600933498806424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,65536,10.822013854980469
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,16384,2.5780745612250433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,65536,11.910063001844618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,10240,1.5815235773722331
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,10240,1.5686630672878688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,8192,1.2585538228352864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,7168,1.069943110148112
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,8192,1.16248713599311
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,7168,1.0033253563774955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,5120,0.7814044422573514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,6144,0.9466782675849067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,6144,0.7702639897664388
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,5120,0.6533777978685167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,4096,0.5833848847283257
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,3584,0.5526568624708388
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,3072,0.4275360107421875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,4096,0.5536977979871962
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,2560,0.4137084484100342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,3072,0.4329644574059381
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,3584,0.5109004444546169
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,1536,0.22269511222839355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,2560,0.39421688185797793
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,2048,0.2880213260650635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,1536,0.26979290114508736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,2048,0.3328026665581597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,1024,0.18049955368041992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,768,0.1457511054144965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,512,0.09481955236858791
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,1024,0.21240533722771537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,768,0.18199377589755586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,256,0.0697937806447347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,128,0.036944889360004
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,256,0.13501066631740993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,64,0.02748888896571265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,128,0.12459999985165066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,512,0.16555110613505045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,32,0.021659556362364028
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,64,0.11802132924397786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,32,0.12184177504645453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,12288,1.4520817862616644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,16384,2.0323236253526473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,16384,2.443752924601237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,65536,9.949239095052084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,10240,1.2478657828436959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,10240,1.5372888776991103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,12288,1.8470399644639757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,8192,1.0488142437405057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,65536,10.355826483832464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,7168,0.8711413277520074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,8192,1.1618195639716256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,6144,0.6807822121514214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,6144,0.812288867102729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,5120,0.6609573364257812
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,7168,0.952180438571506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,4096,0.5034071074591743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,3584,0.3935680124494765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,5120,0.6821529070536295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,3072,0.37708176506890195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,4096,0.5770577854580349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,2560,0.30366399553087026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,2048,0.2698177761501736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,3584,0.5823822021484375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,2560,0.4396391179826524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,3072,0.5235608948601617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,2048,0.40292978286743164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,1024,0.15370933214823404
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,768,0.10650844706429376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,1536,0.22153777546352812
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,512,0.07690488629870944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,256,0.048206223381890186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,1536,0.3521333270602756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,128,0.026672000686327618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,1024,0.288848876953125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,64,0.022700443863868713
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,512,0.2433475653330485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,768,0.26972532272338867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,256,0.2152951161066691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,128,0.20120178328620064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,32,0.20580621560414633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,32,0.01755288905567593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,64,0.19135467211405435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,65536,7.8737089369032125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,12288,1.3337582482231989
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,16384,1.8025538126627605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,12288,1.7997271219889324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,10240,1.315344916449653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,16384,2.4054383171929254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,65536,9.15149180094401
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,8192,0.9773030810885959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,10240,1.5128577550252278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,8192,1.0527093675401475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,7168,0.8233146667480469
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,7168,0.9393350813123914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,5120,0.5749120182461208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,6144,0.7247609032524956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,6144,0.7968844307793511
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,5120,0.6481333308749729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,4096,0.5037751197814941
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,4096,0.5487964418199327
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,3584,0.40811999638875324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,3584,0.4884986877441406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,3072,0.3877217769622803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,2560,0.3366222116682265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,2048,0.2492204507191976
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,3072,0.46530845430162215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,1536,0.19910044140285918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,2560,0.42785154448615176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,1024,0.1340622239642673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,768,0.09581155247158474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,2048,0.3719217777252197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,512,0.06458577844831678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,1536,0.3253520064883762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,256,0.03789155681927999
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,768,0.2531342241499159
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,128,0.02478844424088796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,1024,0.27218932575649685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,512,0.22059999571906197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,64,0.020607110526826646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,32,0.016579555140601266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,256,0.19751732879214814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,128,0.18564089139302573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,32,0.18408621682061088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,64,0.17388444476657441
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,12288,1.014277352227105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,65536,6.949155595567491
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,16384,1.5635031594170465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,10240,0.9489848878648547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,16384,2.1491902669270835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,12288,1.6082969241672094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,10240,1.3514888551500108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,8192,0.7578168974982368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,7168,0.6155929035610622
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,8192,0.9252719879150391
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,6144,0.5030861960517036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,65536,9.16796620686849
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,7168,0.8021590974595812
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,5120,0.4910008642408583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,4096,0.3639350997077094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,5120,0.5748568640814887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,6144,0.715670214758979
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,3584,0.30142754978603786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,3072,0.29083289040459526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,4096,0.47112708621554905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,3072,0.38481420940823025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,3584,0.4219173325432672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,2560,0.2209022177590264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,1536,0.1344453361299303
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,2048,0.18705066045125326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,1024,0.09426044755511814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,2560,0.3603919876946344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,768,0.07752799987792969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,1536,0.27648533715142143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,2048,0.3305262194739448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,512,0.054957333538267344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,256,0.03397866752412584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,1024,0.23669065369500053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,128,0.02017155620786879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,64,0.018761777215533786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,512,0.17938221825493705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,32,0.0138568886452251
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,256,0.15724089410569933
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,768,0.21285422643025717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,128,0.14364622698889837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,64,0.13974577850765652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,32,0.14091022809346518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,65536,6.14333258734809
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,12288,0.8946453730265299
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,16384,1.1930649015638564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,10240,0.8716790941026477
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,12288,1.5775715510050456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,8192,0.6201813485887315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,16384,2.123292499118381
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,10240,1.309443579779731
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,7168,0.5722337828742133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,8192,0.9595279693603516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,6144,0.5190755526224772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,5120,0.4456062316894531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,7168,0.7835209104749891
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,65536,8.213250901963976
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,4096,0.37686755922105575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,3584,0.2928853299882677
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,6144,0.6443182097540961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,5120,0.5330710940890843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,2560,0.22996534241570366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,3072,0.2688853210873074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,4096,0.4384231037563748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,3072,0.35363109906514484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,3584,0.40405599276224774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,2048,0.18494578202565512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,2560,0.31224799156188965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,1536,0.16680444611443412
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,1024,0.09212533632914226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,768,0.07589511076609294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,2048,0.2927617761823866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,512,0.05391110976537069
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,256,0.027613333529896204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,1024,0.2044746610853407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,1536,0.25602844026353627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,768,0.1775519980324639
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,64,0.015191111299726697
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,128,0.01756000022093455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,512,0.14214221636454263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,32,0.01256622208489312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,256,0.1308293342590332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,128,0.1268951098124186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,32,0.11863911151885986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,64,0.12320621808369954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,65536,5.08463372124566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,12288,0.7499546474880643
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,16384,1.0099893146091037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,10240,0.6240951220194498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,8192,0.546595573425293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,12288,1.3611208597819011
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,7168,0.44088088141547305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,16384,1.846743901570638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,10240,1.1402222315470378
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,8192,0.8115928967793783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,7168,0.6193688710530599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,6144,0.3826471169789632
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,5120,0.297598229514228
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,6144,0.4959244198269314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,5120,0.391066657172309
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,4096,0.30776090092129177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,4096,0.23736800087822807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,3584,0.2039439943101671
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,2560,0.14637245072258845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,3072,0.17851466602749297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,3584,0.2678248882293701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,3072,0.2257502343919542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,2048,0.12114222844441731
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,1536,0.09523910946316189
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,2048,0.16775733894772002
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,2560,0.18820177184210884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,1536,0.1353786653942532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,1024,0.07305510838826497
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,768,0.05156088868776957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,1024,0.09774221976598103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,512,0.03681688838534885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,65536,4.0862274169921875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,65536,8.202312893337673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,768,0.09067733420266046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,256,0.022297778063350256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,512,0.07700266440709432
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,64,0.013240888714790344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,32,0.009946666657924652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,256,0.06579377916124132
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,128,0.017070222232076857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,32,0.05674044291178385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,128,0.06040800280041165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,64,0.055212444729275174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,16384,0.8638533486260308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,12288,0.7687466409471301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,10240,0.5682364569769965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,8192,0.5673457781473795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,12288,1.366927146911621
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,16384,1.843109342787001
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,8192,0.7222284211052789
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,7168,0.4553724394904242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,10240,1.1392640007866752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,6144,0.3661964469485813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,5120,0.33266933759053546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,7168,0.606536865234375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,4096,0.24625245730082193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,6144,0.4846684667799208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,4096,0.3116648991902669
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,5120,0.3842808935377333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,3584,0.20240978399912515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,3584,0.26623111300998265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,3072,0.17708088292015922
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,2560,0.15340622266133627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,2560,0.19942311445871988
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,3072,0.2562648985120985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,1536,0.0945075551668803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,2048,0.1308844486872355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,65536,3.460842768351237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,768,0.05105955402056376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,2048,0.14662755860222712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,65536,7.438048468695746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,1024,0.09480533334943984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,512,0.03800088829464383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,1024,0.06982844405704074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,1536,0.12708622879452175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,128,0.015097777048746744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,64,0.011545778148704104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,768,0.0968853301472134
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,32,0.009407111340098912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,256,0.061208003097110324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,512,0.07354399893018934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,256,0.02243288854757945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,64,0.053242666853798755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,128,0.05663555860519409
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,32,0.05541066659821404
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,12288,0.6022817823621962
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,16384,0.7294061978658041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,10240,0.4611111217074924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,8192,0.49530310100979275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,12288,1.2749599880642362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,8192,0.6567893558078343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,7168,0.4070897897084554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,10240,1.054752031962077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,16384,1.7194577323065863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,7168,0.5522978040907118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,6144,0.3364106549157037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,5120,0.3083653450012207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,4096,0.24604445033603242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,3584,0.2235919899410672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,6144,0.4324808915456136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,4096,0.28231999609205455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,5120,0.34389599164326984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,3072,0.19670132795969644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,3584,0.24921600023905435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,2560,0.1508435540729099
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,3072,0.2128702269660102
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,2048,0.11096977525287205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,1536,0.08350933260387844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,1024,0.06991644700368245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,2048,0.12491467263963486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,768,0.042709334029091724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,1536,0.09777866469489203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,1024,0.08276622162924872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,512,0.02919911013709174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,256,0.018225777480337355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,768,0.07155822383032905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,2560,0.1791457732518514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,512,0.0665280024210612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,128,0.012118221984969245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,65536,3.4897333780924478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,64,0.010223111344708337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,256,0.0535715553495619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,32,0.01055555542310079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,64,0.04454311066203647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,32,0.04465333289570279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,128,0.04807644420199924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,65536,7.452329847547744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,16384,0.6775422096252441
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,12288,0.45311821831597227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,10240,0.48320357004801434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,8192,0.37678133116828066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,16384,1.7181670930650499
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,10240,1.0379947026570637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,12288,1.3009191089206271
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,7168,0.31508178181118435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,6144,0.2442951202392578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,8192,0.6788720024956597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,5120,0.25262223349677193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,7168,0.5476142035590278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,6144,0.4279537730746799
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,65536,2.672067642211914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,4096,0.2059617837270101
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,3584,0.1697822279400296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,5120,0.33960888120863175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,3072,0.1511182255215115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,4096,0.2873146798875597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,2560,0.13342844115363225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,3072,0.2055128945244683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,2048,0.11264088418748643
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,3584,0.23904977904425728
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,1536,0.09066133366690741
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,1024,0.06600977977116902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,2560,0.16743200355105928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,768,0.044165333112080894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,2048,0.12311377790239121
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,1536,0.09590044286515977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,512,0.028239998552534316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,256,0.018019555343521964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,1024,0.0752320024702284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,128,0.011952888634469775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,64,0.009756444229020013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,512,0.06493866443634033
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,32,0.008080888953473832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,256,0.049639112419552274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,128,0.04472800095876058
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,768,0.0701999995443556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,64,0.04180622100830078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,32,0.04431466592682732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,65536,6.957680172390408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,12288,0.3305635452270508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,16384,0.4566462304857042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,8192,0.259589327706231
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,10240,0.301092439227634
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,7168,0.20839556058247885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,12288,1.1827999750773113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,10240,0.9622977574666342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,8192,0.5952453083462185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,6144,0.1752328872680664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,7168,0.5036906666225857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,16384,1.5958515803019206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,5120,0.14360177516937256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,5120,0.30040621757507324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,3584,0.10540622472763062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,4096,0.1149839957555135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,65536,2.2173084682888455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,3072,0.08763377534018622
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,6144,0.38546310530768496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,4096,0.2430488798353407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,2560,0.09116444322797988
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,3584,0.2080142233106825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,2048,0.06369600031110975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,3072,0.17388444476657441
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,1536,0.05697511302100288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,2560,0.14333421654171413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,1024,0.04169777697987027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,768,0.03362755642996894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,512,0.02240266733699375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,1536,0.07491199837790595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,2048,0.10662044419182672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,1024,0.06046666701634725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,768,0.055000000529819064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,256,0.012325333224402534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,128,0.008456000023418004
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,64,0.006727111008432176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,32,0.005997333261701796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,256,0.03997600078582764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,128,0.03492266601986355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,512,0.05007288853327433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,64,0.03277333246337043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,32,0.03211466802491082
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,12288,0.2627760039435493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,16384,0.37627021471659344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,10240,0.23525423473782012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,65536,6.89724604288737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,8192,0.19621866279178193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,65536,1.4947946336534288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,7168,0.2253048949771457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,12288,1.1792631149291992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,16384,1.6082462734646266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,10240,0.9676230748494467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,6144,0.16456622547573513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,5120,0.13881422413720026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,8192,0.5827191140916612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,4096,0.13015466266208225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,7168,0.5053324699401855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,6144,0.38143110275268555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,3584,0.11444000403086345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,5120,0.2950168980492486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,3072,0.10282578070958455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,4096,0.23541421360439727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,2560,0.08710044622421265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,3072,0.16506755352020264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,2048,0.07895288864771526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,3584,0.210589329401652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,1536,0.05089866783883837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,1024,0.033999999364217125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,2560,0.13056888845231798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,768,0.02784444557295905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,2048,0.09137422508663601
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,512,0.01941422290272183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,1536,0.07092533508936565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,256,0.011874666644467248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,128,0.008081778056091731
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,1024,0.05761866437064277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,768,0.0500311123000251
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,64,0.006188444379303191
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,512,0.04336177640491062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,32,0.0052888890107472735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,256,0.03621244430541992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,64,0.029900444878472224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,128,0.03165955675972833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,32,0.02901600135697259
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,16384,0.23430132865905762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,12288,0.17722933822207979
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,10240,0.15689778327941895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,8192,0.12870221667819554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,65536,6.408616807725694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,7168,0.10568977726830377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,65536,1.0387599733140733
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,6144,0.1120977799097697
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,8192,0.5192826588948568
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,12288,1.0876266691419814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,10240,0.8896026611328125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,5120,0.07377333111233182
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,4096,0.0648604432741801
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,6144,0.3342106607225206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,7168,0.4382942252688938
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,3584,0.05396355523003472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,5120,0.24398220909966362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,16384,1.4839910931057398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,3072,0.04661688870853848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,2560,0.04065422217051188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,4096,0.19321689340803358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,2048,0.03387022349569533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,3072,0.12834489345550537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,1536,0.02959200077586704
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,2048,0.06638844145668878
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,1024,0.019896000623703003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,1536,0.05254222287072075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,2560,0.09771911303202312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,3584,0.15685333145989314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,768,0.016069332758585613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,512,0.01105955574247572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,1024,0.041344000233544245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,256,0.008044444852405125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,128,0.005511999958091312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,768,0.03625511129697164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,64,0.004986666556861666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,512,0.031368888086742826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,256,0.026013332936498854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,32,0.0039822223285834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,128,0.022454222043355305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,64,0.02055111030737559
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,32,0.02083288960986667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,16384,0.1954400009579129
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,12288,0.15934488508436415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,10240,0.12960978349049887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,8192,0.10705600182215373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,65536,0.8254453341166178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,7168,0.11176710658603245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,8192,0.49556710984971786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,10240,0.8821875784132215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,12288,1.080151981777615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,65536,6.4064453972710504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,6144,0.08496444755130345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,5120,0.07243555784225464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,4096,0.05827377902136909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,3584,0.051366223229302295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,6144,0.314102225833469
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,16384,1.4708177778455946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,4096,0.16617688867780897
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,5120,0.2379279931386312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,3072,0.044983112149768405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,7168,0.4145413239796956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,2560,0.03801066676775614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,3072,0.10476444164911906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,2048,0.03310666812790765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,3584,0.1304817729526096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,2560,0.0796275536219279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,2048,0.060462223158942334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,1024,0.018352889352374606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,1536,0.026030222574869793
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,512,0.010840888652536603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,768,0.014915555715560913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,1536,0.0496053331428104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,256,0.007205333146784041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,768,0.034155554241604276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,1024,0.0384808878103892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,128,0.005304000029961268
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,512,0.028631998433007136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,64,0.004411555412742826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,256,0.024023110667864483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,32,0.0038159998754660287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,128,0.020418667131000094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,64,0.01900622248649597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,32,0.0181688881582684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,12288,0.09273688660727607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,16384,0.13488799995846218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,10240,0.08099999692704943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,65536,0.5855893558926052
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,8192,0.06968888971540663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,10240,0.8722782135009766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,8192,0.5015137990315756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,7168,0.06365333663092719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,12288,1.071533309088813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,6144,0.05289599961704678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,65536,5.9473876953125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,16384,1.4597981770833333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,5120,0.04466933343145582
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,4096,0.036849776903788246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,3584,0.033519112401538424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,3072,0.03314222229851617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,4096,0.1423919995625814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,6144,0.29212265544467503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,7168,0.408202674653795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,2560,0.02873777680926853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,3584,0.11338488260904948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,5120,0.22004711627960205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,1536,0.018245332770877414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,3072,0.08647466368145412
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,2048,0.02201066745652093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,1024,0.012060444388124677
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,2560,0.06735377841525607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,768,0.010111110905806223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,2048,0.05611733595530192
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,512,0.007859555383523306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,256,0.00544088871942626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,1536,0.04626311196221245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,1024,0.036095112562179565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,768,0.031361778577168785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,128,0.0042071110672420925
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,512,0.026526222626368206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,64,0.003474666840500302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,32,0.002944888960984018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,256,0.021361778179804485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,128,0.01806400054030948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,64,0.01680355601840549
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,32,0.016255110502243042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,16384,0.06713777780532837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,12288,0.04933866527345446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,10240,0.04187822341918945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,8192,0.0350560016102261
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,65536,0.5046453475952148
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,7168,0.031050665511025324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,8192,0.5031804508633083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,6144,0.027795556518766616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,7168,0.3991244369082981
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,10240,0.861198213365343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,5120,0.024550222688251074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,4096,0.020562666985723708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,12288,1.0618267059326172
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,6144,0.27864623069763184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,3584,0.018780445059140522
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,5120,0.18695999516381157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,65536,5.917518191867405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,3072,0.01720533271630605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,4096,0.12013333373599583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,16384,1.4634168412950304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,3584,0.09307199716567993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,2048,0.01586933268441094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,2560,0.018623999423450895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,1536,0.01237422227859497
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,3072,0.07566310962041219
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,2560,0.06576089064280193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,1024,0.009023110899660323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,768,0.007500444849332173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,2048,0.054011556837293834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,1536,0.04407733347680834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,512,0.005971555494599872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,1024,0.03418755531311035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,768,0.029709332519107397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,256,0.004436444491147995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,128,0.003738666574160258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,512,0.024544888072543677
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,64,0.0029484445436133277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,256,0.019498666127522785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,32,0.0027386666172080566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,128,0.01650044487582313
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,64,0.014984889162911309
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,32,0.01440177857875824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,16384,0.03513333201408386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,12288,0.027419555518362258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,10240,0.02898755669593811
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,65536,0.44975821177164715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,8192,0.023860444625218708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,10240,0.8514649073282877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,7168,0.02216266592343648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,8192,0.5048053529527452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,12288,1.057472864786784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,6144,0.019536000159051683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,5120,0.016924444172117446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,16384,1.454427507188585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,7168,0.3875066704220242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,5120,0.17900710635715059
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,4096,0.014064000712500678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,6144,0.254223108291626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,65536,5.9115388658311625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,3584,0.013368888861603208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,3072,0.015257777439223396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,2560,0.013618666264745923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,3584,0.08807200193405151
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,4096,0.10445066955354478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,2048,0.012051555845472546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,1536,0.008445333275530074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,3072,0.07466755310694377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,2560,0.06411733229955037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,1024,0.0063777777055899305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,768,0.005621333503060871
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,1536,0.04400711258252462
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,2048,0.05377777748637729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,512,0.004563555535342959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,128,0.003335111050142182
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,256,0.0036319999231232535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,1024,0.03376888897683885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,512,0.02397866712676154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,768,0.029311110575993855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,64,0.0026435556097163092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,256,0.01919644408755832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,32,0.0024071110205517872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,128,0.016320889194806416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,32,0.014427555931939019
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,16384,0.03262933426433139
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,64,0.014879999889267815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,12288,0.026211554805437725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,10240,0.022868444522221882
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,8192,0.01830666595035129
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,65536,0.4378968874613444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,7168,0.03641422258483039
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,8192,0.4927475717332628
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,7168,0.38146310382419163
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,6144,0.03109244505564372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,5120,0.026504889130592346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,10240,0.8467369079589844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,4096,0.020991999242040846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,12288,1.057294209798177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,65536,5.900752173529731
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,6144,0.2510160075293647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,16384,1.4508017434014215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,3584,0.019631110959582858
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,5120,0.16660443941752115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,2560,0.01424088908566369
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,3072,0.016883555385801528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,2048,0.01148977792925305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,4096,0.10296533505121867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,3584,0.08754044771194458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,1536,0.009375111096435124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,2560,0.06360799736446805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,2048,0.05367911193105909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,3072,0.07403910822338529
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,768,0.0057848890622456866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,1024,0.006811555474996567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,1536,0.04384533233112759
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,1024,0.033647110064824425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,256,0.0033457777980301115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,512,0.004448889030350579
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,128,0.0027964444210131965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,768,0.02903111113442315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,64,0.0022195554855797025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,512,0.023805333508385554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,256,0.01904622217019399
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,32,0.0021359999146726397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,128,0.01625066664483812
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,64,0.014837332897716098
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,32,0.01440088947614034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,16384,0.03230311142073737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,12288,0.026944888962639704
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,10240,0.02292355563905504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,8192,0.017442666822009616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,65536,0.3835502200656467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,8192,0.49363554848565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,10240,0.8530337545606824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,7168,0.020949333906173706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,6144,0.018089777893490262
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,12288,1.0575670666164823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,5120,0.013270222478442721
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,65536,5.893039279513889
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,7168,0.38424356778462726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,6144,0.2433617909749349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,16384,1.455344835917155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,4096,0.010988444089889526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,3584,0.010489777558379702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,5120,0.16624622874789768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,3072,0.00943288869327969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,2560,0.008719111482302347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,4096,0.10327644480599298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,2048,0.0074755557709270054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,3584,0.08804088830947876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,1536,0.00889244427283605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,3072,0.07383644580841064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,2560,0.06373333268695407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,1024,0.006227555374304454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,768,0.005212444398138259
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,512,0.0041457778877682155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,2048,0.05348177750905355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,1536,0.043735110097461276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,1024,0.033647110064824425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,768,0.02894044584698147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,256,0.003105777833196852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,128,0.002502222235004107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,64,0.0021422221842739317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,512,0.023927999867333308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,256,0.01901422275437249
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,32,0.0020986667109860317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,128,0.016241777274343703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,64,0.014856000741322836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,32,0.0144195556640625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,65536,5.908155653211806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,12288,3.293357425265842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,10240,2.7476577758789062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,65536,5.936547597249349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,16384,4.34788343641493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,10240,5.139385647243924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,8192,2.266675525241428
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,7168,1.9942745632595484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,8192,4.157800886366102
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,12288,6.150181240505642
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,6144,1.7452320522732203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,6144,3.0704125298394094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,7168,3.5466639200846353
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,4096,1.2549031575520833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,5120,1.472492429945204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,4096,2.03859731886122
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,16384,8.210243225097656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,5120,2.6304666731092667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,3072,0.9973537656995984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,3584,1.1184790929158528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,2560,1.2052408854166667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,3584,1.787064022488064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,3072,1.485189331902398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,2560,0.9059635798136393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,2048,0.995107544793023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,2048,0.781800005171034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,1024,0.5464408662584093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,1024,0.6162595748901367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,1536,0.7108951144748263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,768,0.5544933213127984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,512,0.40174134572347003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,1536,0.6552809079488119
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,256,0.2761404514312744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,64,0.2155359983444214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,512,0.44932179980807835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,128,0.26459911134507924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,768,0.5153324339124892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,128,0.41295377413431805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,64,0.3420417838626438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,256,0.40595377816094297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,32,0.39115201102362734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,32,0.26170489523145885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,12288,1.0734684202406142
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,10240,0.8397750854492188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,16384,1.4177973005506728
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,10240,1.2748346328735352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,12288,1.5736310746934679
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,16384,2.115247938368056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,8192,0.6763057708740234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,7168,0.6585600111219618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,6144,0.7390471034579806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,8192,0.9860764609442817
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,7168,0.8710177739461263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,6144,0.5851084391276041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,5120,0.47751108805338544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,4096,0.4160791238149007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,3584,0.4581644270155165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,5120,0.570359124077691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,4096,0.5089093314276801
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,2560,0.3365608851114909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,2560,0.2980755435095893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,3584,0.36717955271402997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,3072,0.3611591127183702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,2048,0.2806284427642822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,1536,0.1861057811313205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,3072,0.3218657705518934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,1024,0.14272356033325195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,1536,0.21285155084398058
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,512,0.07434399922688802
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,2048,0.2485804557800293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,768,0.10219199789894952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,768,0.16607466008928087
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,256,0.04103644357787238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,128,0.027698665857315063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,64,0.02269866731431749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,256,0.111063109503852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,512,0.13252088758680555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,128,0.10267821947733562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,1024,0.16775643825531006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,32,0.017791999710930716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,64,0.09916444619496663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,32,0.09734844499164158
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,12288,1.1472667058308919
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,16384,1.1916542053222656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,16384,1.5536186430189345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,12288,0.857218689388699
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,10240,0.9202746285332574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,10240,0.7160293261210123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,65536,6.624362521701389
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,8192,0.7023484441969129
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,8192,0.5770764350891113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,7168,0.6621075736151801
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,6144,0.5025688807169596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,6144,0.44240710470411515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,7168,0.49028624428643125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,5120,0.45221508873833555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,4096,0.3859199947781033
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,3584,0.30776090092129177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,5120,0.3947422239515517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,3072,0.2883404360877143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,4096,0.3165164523654514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,3584,0.3123137685987684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,65536,8.387764824761286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,3072,0.263500452041626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,2560,0.23559644487169054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,1536,0.16074666712019178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,2048,0.19182399908701578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,2560,0.21850932968987358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,1024,0.09689066807428996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,768,0.07844977908664279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,512,0.05574044254091051
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,1024,0.14470399750603571
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,2048,0.18125600285000273
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,256,0.03184355629814996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,128,0.021054221524132624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,768,0.11812799506717259
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,64,0.01795733306143019
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,512,0.10467288891474406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,1536,0.1562195618947347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,32,0.014074666632546319
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,128,0.08398222261004978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,64,0.0783368878894382
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,256,0.09121421972910564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,32,0.07918044593599108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,65536,5.592861175537109
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,12288,0.7674702008565267
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,12288,0.8981466293334961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,16384,1.0294053819444444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,16384,1.264569812350803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,65536,6.317888895670573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,10240,0.7385030852423774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,10240,0.6357946925693089
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,8192,0.6169795460171169
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,8192,0.47594308853149414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,7168,0.6625964376661513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,7168,0.44849242104424375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,6144,0.3811475435892741
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,5120,0.314302232530382
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,5120,0.4828977584838867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,4096,0.28177865346272785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,4096,0.355055120256212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,3584,0.3009173340267605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,6144,0.5249528884887695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,2560,0.2384746604495578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,3072,0.22321600384182402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,3584,0.24536177847120497
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,2560,0.18978399700588652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,3072,0.2735839949713813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,2048,0.1920266681247287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,2048,0.1621333360671997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,1024,0.10914844274520874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,1536,0.16154221693674722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,1536,0.13519822226630315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,512,0.04672266708479988
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,256,0.02830577890078227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,768,0.07033689154518975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,768,0.10098222229215835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,1024,0.11167200406392415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,512,0.08854489194022284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,128,0.01755466726091173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,64,0.01566133399804433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,32,0.012535110943847232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,256,0.07802666558159722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,64,0.06592000193066068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,32,0.06695466571384005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,128,0.0726755526330736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,12288,0.663914680480957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,16384,0.9878098169962565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,16384,0.9924870596991645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,65536,4.898450639512804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,10240,0.6561626858181423
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,12288,0.7208008766174316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,8192,0.5206640031602647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,65536,5.100147671169705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,10240,0.5849777857462565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,7168,0.40001598993937176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,8192,0.491671085357666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,6144,0.3450488779279921
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,7168,0.42851466602749294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,6144,0.3724657694498698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,5120,0.302877320183648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,4096,0.24888711505466035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,3584,0.19569067160288492
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,3072,0.17543999354044595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,2560,0.1434346702363756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,5120,0.3397786617279053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,3072,0.26039910316467285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,4096,0.2854471206665039
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,2048,0.16531111134423151
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,3584,0.28875290022956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,2048,0.11594488885667588
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,2560,0.2306871149275038
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,1024,0.06285778019163343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,768,0.050494223833084106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,1536,0.16421511438157824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,1536,0.09324977795283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,512,0.03667111198107401
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,1024,0.12419644991556804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,256,0.024089778463045757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,128,0.014523555835088095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,512,0.10497066709730361
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,32,0.010135110881593492
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,256,0.09938489066229926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,768,0.1177342202928331
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,64,0.01328000012371275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,64,0.09026755226982965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,128,0.08921511305703057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,32,0.0905733307202657
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,65536,4.308368682861328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,12288,0.5797031190660265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,65536,4.573983934190538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,16384,0.7906444337632922
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,10240,0.5354471206665039
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,12288,0.6898969014485677
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,16384,0.9880257712470161
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,8192,0.4785182211134169
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,10240,0.5987173186408149
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,7168,0.4309377670288086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,8192,0.46251556608412003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,6144,0.33361156781514484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,7168,0.39644622802734375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,4096,0.2864320013258192
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,4096,0.22920534345838758
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,3584,0.19603555732303193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,5120,0.2844959894816081
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,6144,0.3893253273434109
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,5120,0.3466293281979031
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,2560,0.14460444450378418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,3584,0.23193155394660103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,3072,0.23752710554334852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,3072,0.17101332876417372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,2048,0.12833600574069554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,2560,0.2059946722454495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,1024,0.06242844131257799
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,2048,0.1483902268939548
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,1536,0.10336177878909641
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,768,0.05010399884647793
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,512,0.03631466627120972
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,256,0.023294222023752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,1024,0.11343555980258518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,768,0.09421244594785903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,1536,0.14623555872175428
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,128,0.015077332655588785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,512,0.08710755242241754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,32,0.00943022221326828
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,64,0.011566222541862063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,64,0.0772577789094713
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,32,0.07351111041174994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,128,0.08119377824995253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,256,0.08207911252975464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,12288,0.5460649066501194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,16384,0.7195617887708875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,12288,0.6666328642103407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,65536,3.9048843383789062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,16384,0.9695271386040581
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,10240,0.5100720193650987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,8192,0.43773688210381406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,10240,0.5478649139404297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,7168,0.388117339875963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,8192,0.46898486879136825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,6144,0.3432888984680176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,7168,0.41890043682522243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,5120,0.29081688986884224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,65536,4.532176123725043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,6144,0.3694177733527289
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,4096,0.22558400366041395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,5120,0.32345599598354763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,3584,0.22014489438798693
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,3072,0.1721902158525255
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,3072,0.19209422005547416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,2560,0.15191022555033365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,2048,0.13011733690897623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,3584,0.2210693359375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,4096,0.2571377754211426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,1536,0.0871608919567532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,2560,0.19139911068810356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,1536,0.12036177847120498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,2048,0.14477777481079102
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,1024,0.05079466766781277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,1024,0.09820799695120917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,768,0.03972533345222473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,512,0.029189334975348577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,768,0.0901342233022054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,128,0.012364444633324942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,512,0.0811671084827847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,256,0.0741457806693183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,64,0.010039110978444418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,65536,3.5302658081054688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,32,0.008377778033415476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,128,0.07117066780726115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,256,0.019102222389645047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,32,0.06446844339370728
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,64,0.06834133466084798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,12288,0.44980091518825954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,16384,0.6115333239237467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,12288,0.680973317888048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,65536,4.5223388671875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,10240,0.4046302106645372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,16384,0.9961333804660373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,10240,0.5608782238430446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,8192,0.3128800127241346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,7168,0.29071911176045734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,5120,0.23111645380655924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,8192,0.45160531997680664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,6144,0.2401342127058241
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,7168,0.4041706720987956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,4096,0.19549155235290527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,6144,0.3622639973958333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,5120,0.31562754842970103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,3584,0.1585751109653049
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,3072,0.15291821956634521
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,2560,0.11441955301496719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,3072,0.19129155741797554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,4096,0.25436533821953666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,2560,0.16638399495018855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,3584,0.2217715581258138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,1536,0.08987289004855686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,1024,0.05451822280883789
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,2048,0.09242488940556844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,768,0.04013333386845059
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,1024,0.09498755799399482
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,1536,0.11499288347032334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,2048,0.1402133305867513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,512,0.029669334491093952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,65536,2.769673877292209
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,256,0.017496888836224873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,128,0.012180444267061023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,64,0.010773333410422007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,32,0.008119111259778341
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,512,0.07463822099897596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,256,0.07104266352123685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,128,0.06353866391711764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,768,0.08747644556893243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,64,0.061423109637366406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,32,0.06303289201524523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,12288,0.33254755867852104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,10240,0.3071066538492839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,16384,0.4569884406195746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,12288,0.45666223102145725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,8192,0.2664159933725993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,16384,0.6938080257839627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,7168,0.190628449122111
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,10240,0.3758675522274441
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,8192,0.31640977329678005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,6144,0.16524622175428602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,7168,0.27080978287590873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,5120,0.14767821629842123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,4096,0.11601244078742133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,6144,0.2291395664215088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,3584,0.09892621967527603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,5120,0.1939244402779473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,4096,0.13521422280205622
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,3072,0.0860640009244283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,2560,0.07811466852823894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,2048,0.05847644143634372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,3072,0.09934755828645493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,2560,0.08865511417388916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,65536,4.517449696858724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,2048,0.07507999738057454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,1536,0.04611466659439934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,3584,0.1134035587310791
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,1024,0.03308088911904229
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,768,0.028566221396128338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,512,0.020942222740915086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,256,0.01293511109219657
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,1024,0.05490844448407491
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,65536,2.1168212890625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,1536,0.06462133593029447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,512,0.04393600092993843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,128,0.008431110945012834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,64,0.006736889067623351
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,768,0.04904266529613071
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,256,0.03788355655140347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,64,0.032038221756617226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,32,0.03178222311867608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,128,0.039911111195882164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,32,0.006064888917737537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,16384,0.4519724316067166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,12288,0.35612800386216903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,10240,0.29492799441019696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,16384,0.666137801276313
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,8192,0.24411111407809785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,12288,0.457839118109809
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,10240,0.3769555621676975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,8192,0.29736267195807564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,7168,0.20498843987782797
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,5120,0.14502399497561985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,7168,0.2573217815823025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,65536,3.6978001064724393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,6144,0.16689333650800917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,65536,1.8675778706868489
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,6144,0.2201688951916165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,3584,0.09893866380055745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,4096,0.11371378103892009
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,2560,0.07267822159661187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,3072,0.08668000168270534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,3072,0.09612533118989731
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,3584,0.12607644663916692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,5120,0.17977421813540986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,4096,0.12735999955071345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,2048,0.05845955345365736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,2560,0.08419022295210098
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,1536,0.04810133245256212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,1024,0.03439377744992574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,1024,0.052276445759667285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,768,0.028146667612923518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,1536,0.06271288792292277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,512,0.020118221640586853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,256,0.0123831108212471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,128,0.00814844419558843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,2048,0.07246577739715576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,768,0.04885777831077576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,64,0.0063884444534778595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,32,0.0058266665372583605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,512,0.04188799858093262
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,64,0.030551112360424463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,32,0.030260443687438965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,256,0.03618311219745212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,128,0.03195910983615451
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,10240,0.2307493421766493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,12288,0.26447733243306476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,16384,0.34137243694729275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,8192,0.1876373291015625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,12288,0.4630595313178168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,8192,0.29422399732801646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,16384,0.6744044621785482
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,10240,0.36737243334452313
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,7168,0.18537244531843397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,7168,0.25116443634033203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,65536,3.6728185017903647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,6144,0.1643084420098199
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,5120,0.1387608846028646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,6144,0.21517511208852133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,4096,0.12245244450039333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,65536,1.5619653065999348
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,3584,0.10934488640891181
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,5120,0.1768497758441501
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,3072,0.09893777635362412
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,4096,0.1227946678797404
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,3584,0.10836622450086807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,2048,0.0589324434598287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,2560,0.10505244466993545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,1536,0.04457599918047587
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,3072,0.09312888648774888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,2560,0.08207644356621636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,1024,0.03418577710787455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,512,0.01976355579164293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,2048,0.07040177451239692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,1536,0.06062666575113932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,768,0.027036445008383855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,256,0.011956444217099084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,128,0.00812888890504837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,512,0.03987911012437608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,64,0.006249777972698212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,768,0.045147554741965405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,256,0.03439199924468994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,1024,0.055065777566697865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,128,0.030810667408837214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,32,0.005241777747869492
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,64,0.029157333903842505
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,32,0.02861777941385905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,12288,0.2281431092156304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,10240,0.19548355208502877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,16384,0.3090382152133518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,12288,0.4602986441718207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,8192,0.16324888335333929
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,16384,0.6917724609375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,10240,0.3656204541524251
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,7168,0.16720267136891684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,8192,0.2883431116739909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,6144,0.13922311200035944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,7168,0.2483688990275065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,5120,0.14689689212375218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,65536,1.3669777976142035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,4096,0.10291288958655463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,6144,0.21098044183519152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,3584,0.10257689158121745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,4096,0.12072532706790501
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,3584,0.1062213314904107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,3072,0.08552889029184978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,5120,0.16947733031378853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,2560,0.07623822159237333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,2048,0.06558310985565186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,1536,0.048651556173960366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,2048,0.06906133227878146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,65536,3.6786774529351125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,2560,0.08034221993552314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,1024,0.03382044368320041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,3072,0.099199997054206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,1536,0.059380445215437144
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,512,0.019315555691719055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,1024,0.0484773317972819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,256,0.012667555775907306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,768,0.043677333328458995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,768,0.02658222284581926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,64,0.006153777655627992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,128,0.008060444560315874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,256,0.033055110110176936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,512,0.03849955399831136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,128,0.03054311209254795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,32,0.02790133489502801
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,64,0.028041776683595445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,32,0.004972444640265571
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,16384,0.23187022738986543
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,12288,0.19951911767323813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,10240,0.15716532866160074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,10240,0.28793332311842174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,8192,0.12789421611362034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,12288,0.37717334429423016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,16384,0.5710106425815159
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,7168,0.10263199938668145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,6144,0.09313333034515381
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,65536,1.0411840014987521
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,8192,0.2209999958674113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,7168,0.18915911515553793
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,5120,0.07245688968234591
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,4096,0.07395288679334852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,3072,0.057322667704688184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,3584,0.05500266618198819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,5120,0.12368088298373753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,4096,0.0856035550435384
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,3584,0.071706665886773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,6144,0.15476622846391466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,2560,0.03982933362325033
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,2048,0.032630221711264715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,1536,0.02791911032464769
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,1024,0.020071110791630216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,2560,0.056051552295684814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,768,0.016509332590632968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,1536,0.04240533378389147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,2048,0.0478168891535865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,1024,0.03467466764979892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,512,0.012170666621790992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,3072,0.07172177897559272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,65536,3.6678140428331165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,256,0.007376000285148621
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,128,0.005582222094138463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,768,0.032067557175954185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,512,0.028247111373477515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,32,0.003959111041492886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,256,0.023972445064120825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,64,0.02011644509103563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,128,0.02216622233390808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,32,0.0203484445810318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,64,0.00464088883664873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,12288,0.16118133068084717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,16384,0.19117244084676108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,10240,0.13052977455986872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,8192,0.1116595533159044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,12288,0.34990665647718644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,7168,0.09853155745400323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,10240,0.2633146709865994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,8192,0.1972479952706231
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,16384,0.5802417861090766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,6144,0.08581778075959946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,65536,0.8201182153489855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,5120,0.07179911269081964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,4096,0.05751199854744805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,7168,0.16627821657392713
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,6144,0.13159199555714926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,3584,0.05105688836839464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,5120,0.1031493345896403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,3072,0.04519911275969612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,4096,0.08399111032485962
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,2560,0.037823110818862915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,2048,0.033929778469933405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,3072,0.05877777602937487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,3584,0.0662826697031657
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,1536,0.02801333202256097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,2048,0.04543911086188423
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,1024,0.01826400061448415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,2560,0.05214133196406894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,1536,0.038839999172422625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,768,0.014793778459231058
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,512,0.010989333192507425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,256,0.007147555549939473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,128,0.005294222384691238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,768,0.02907199992073907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,1024,0.03400266501638625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,512,0.025291555457644995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,256,0.021368000242445204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,64,0.004391111019584867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,65536,3.2106880611843533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,32,0.0037635556525654266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,128,0.019181332654423185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,64,0.019533332851197984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,32,0.017961778574519686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,16384,0.12206489510006374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,12288,0.09346310959921943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,10240,0.07942133479648165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,8192,0.0627982219060262
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,65536,0.5137928856743706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,10240,0.2367546558380127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,12288,0.32466310924953884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,7168,0.07034311029646131
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,6144,0.05037066671583387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,16384,0.5638453165690104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,7168,0.1392835511101617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,8192,0.17204977406395805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,5120,0.043876445955700345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,3584,0.03324888812171088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,4096,0.036983999941084124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,5120,0.08635555373297797
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,3072,0.0320720010333591
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,4096,0.06982400019963582
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,6144,0.11445244153340657
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,2560,0.027278222971492346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,3584,0.06313333246443006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,2048,0.02273066673013899
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,3072,0.056371556388007275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,1536,0.01752800080511305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,1024,0.012422222230169507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,2560,0.049802667564815946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,2048,0.04308355516857571
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,1536,0.03718222181002299
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,768,0.010232888989978367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,512,0.007828444242477417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,256,0.005715555614895291
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,128,0.004536889079544279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,1024,0.029658665259679157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,768,0.026545777916908264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,512,0.023295111126369897
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,64,0.0035439998739295532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,32,0.003543111185232798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,128,0.017284444636768766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,64,0.016180444094869826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,65536,3.169113794962565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,32,0.01592266725169288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,256,0.01982133256064521
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,16384,0.09878755278057522
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,12288,0.07518221934636433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,10240,0.07065600156784058
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,65536,0.42540444268120664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,8192,0.052300443251927696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,7168,0.04797866609361437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,10240,0.23636444409688315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,12288,0.3276755544874403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,8192,0.16148177782694498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,7168,0.12832089265187582
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,5120,0.03650133477316962
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,16384,0.560731569925944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,6144,0.04335200124316745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,4096,0.030927111705144245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,3584,0.0310479998588562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,6144,0.10536266697777642
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,3072,0.02700177828470866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,5120,0.08337599701351589
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,4096,0.06889155838224623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,3584,0.06296800242529975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,2560,0.02293866707219018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,2048,0.01777866648303138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,1536,0.014763555592960782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,1024,0.010749333434634738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,3072,0.055099556843439736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,768,0.008918222453859117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,2048,0.041936000188191734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,2560,0.04845066534148323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,1536,0.035197334157096014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,512,0.006814222368929122
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,256,0.005179555465777715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,1024,0.028763555818133887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,128,0.0039822223285834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,768,0.02535022298494975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,512,0.022103110949198406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,64,0.0032622222271230486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,256,0.018216888109842937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,32,0.003112888791494899
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,128,0.01612711118327247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,64,0.015247111519177755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,32,0.015071999695565967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,16384,0.061984890037112765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,12288,0.04937066634496053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,10240,0.04022044605678982
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,65536,3.1674319373236766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,65536,0.3525991174909804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,8192,0.03495644529660543
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,12288,0.30027998818291557
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,7168,0.030674666166305542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,10240,0.21958933936225045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,6144,0.02738933265209198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,8192,0.15046666728125677
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,16384,0.5589600139194065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,5120,0.023870221442646448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,4096,0.020255999432669747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,3584,0.019113777412308585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,7168,0.12029777632819282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,6144,0.09635644488864475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,4096,0.06753689050674438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,3072,0.017127111554145813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,5120,0.08140622244940864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,2560,0.015570667054918079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,3584,0.060989333523644336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,2048,0.01572355628013611
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,1536,0.012186666329701742
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,2560,0.047910223404566445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,3072,0.054670221275753446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,1024,0.008947555389669206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,2048,0.040806220637427434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,768,0.007448888487286038
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,1536,0.03461511267556085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,512,0.005978666659858491
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,256,0.004455111092991299
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,1024,0.027828445037206013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,768,0.024494222468800012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,128,0.0037155557009908888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,512,0.021365332934591506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,256,0.017509332961506315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,64,0.002944888960984018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,32,0.0027671110712819626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,128,0.015499555402331881
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,64,0.01445777714252472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,16384,0.033542222446865506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,32,0.014117333624098035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,12288,0.027229333917299908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,65536,0.27318400806850857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,10240,0.023618666662110224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,8192,0.020398222737842135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,65536,3.1643714904785156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,7168,0.01904533306757609
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,10240,0.21127555105421278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,12288,0.28797867563035756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,8192,0.13635378413730198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,6144,0.017317333155208163
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,16384,0.5472942458258735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,5120,0.017304889029926725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,7168,0.11308088567521836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,6144,0.09539110793007745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,4096,0.014202665951516895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,5120,0.08095110787285699
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,3584,0.013622221847375235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,4096,0.06760266754362318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,3072,0.012432888978057437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,2560,0.011307555768224927
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,2048,0.00961511085430781
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,3584,0.06085955434375339
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,1536,0.008421333299742805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,1024,0.0064275554484791225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,3072,0.05429244372579786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,2560,0.047749333911471896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,2048,0.04087288843260871
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,512,0.004552888787455029
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,1536,0.03451733456717597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,768,0.005586666779385672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,1024,0.027485332555241052
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,256,0.0035777779089079965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,128,0.0030657777355776895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,768,0.024493333366182115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,512,0.02104622291194068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,64,0.00249777775671747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,256,0.01735111077626546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,32,0.0023102222217453853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,128,0.015442666080262927
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,64,0.014498665928840637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,16384,0.023765333824687537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,32,0.014310222533014087
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,12288,0.01947111056910621
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,10240,0.016992888516849942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,8192,0.016688000824716356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,65536,0.24536532825893828
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,7168,0.015424888994958667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,65536,3.170045428805881
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,10240,0.20060622692108154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,8192,0.12940355141957602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,12288,0.2983955542246501
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,7168,0.11122222741444905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,5120,0.012327111429638334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,6144,0.01388533330625958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,16384,0.5488213433159722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,6144,0.09595644474029541
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,5120,0.0807911091380649
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,4096,0.010083555347389644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,4096,0.06763111220465766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,3584,0.009760888914267222
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,3584,0.061167114310794406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,3072,0.008664000365469191
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,3072,0.05446844299634298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,2048,0.011564444336626263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,2560,0.04758044415050083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,2560,0.007911110917727152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,1536,0.009343999955389235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,1024,0.006820444431569841
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,2048,0.04067644476890564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,768,0.005690666536490123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,1536,0.03460888730155097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,1024,0.027683556079864502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,512,0.004395555704832077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,256,0.003264888914095031
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,768,0.024361777636739943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,128,0.0026382222357723448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,512,0.02109688851568434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,64,0.002209777840309673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,256,0.017442666822009616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,32,0.0020924444413847392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,128,0.015623999966515435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,64,0.014482667048772177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,32,0.014183999763594734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,16384,0.031467556953430176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,12288,0.025867554876539443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,10240,0.022082666556040447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,65536,0.20869244469536674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,8192,0.0174666659699546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,12288,0.2823439968956841
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,10240,0.19659022490183511
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,65536,3.180481804741753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,8192,0.12801778316497803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,7168,0.01681688924630483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,6144,0.014759999182489185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,5120,0.012986666626400419
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,16384,0.5484977828131782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,4096,0.010868444210953183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,7168,0.1099439991845025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,3584,0.01036977767944336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,6144,0.09536888864305283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,3072,0.009464888936943477
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,4096,0.06719199816385905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,5120,0.08109511269463433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,2560,0.008651555412345463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,3584,0.06073511309093899
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,2048,0.007479111353556315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,1536,0.008916444248623317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,3072,0.054229332341088184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,1024,0.006594666590293248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,2048,0.04090311129887899
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,2560,0.04738577869203356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,768,0.005381333331267039
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,1536,0.0344195564587911
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,512,0.0041626666982968645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,256,0.00300799993177255
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,1024,0.02753955622514089
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,128,0.002424888933698336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,768,0.02440799938307868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,512,0.02075822154680888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,64,0.002036444428894255
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,32,0.001855111163523462
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,256,0.01711466742886437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,128,0.015629332926538255
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,16384,0.017335999343130324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,64,0.014641778336630927
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,32,0.014166222678290473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,12288,0.014702222413486905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,10240,0.012870222330093384
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,65536,0.19975022474924722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,8192,0.010797333386209277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,7168,0.011853333148691388
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,8192,0.1288835604985555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,10240,0.19582577546437582
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,12288,0.28303999370998806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,7168,0.1106568906042311
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,6144,0.010620444185203977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,65536,3.1769786410861545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,4096,0.008112000094519721
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,5120,0.009496889180607265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,3584,0.007631999750932057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,16384,0.5471599896748861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,6144,0.09532799985673691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,3072,0.0070088886552386815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,5120,0.0807937781016032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,2560,0.00639111093348927
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,3584,0.06104444132910835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,2048,0.005700444595681296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,4096,0.06774044699139066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,1536,0.00775733341773351
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,3072,0.05422400103674995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,2560,0.04763466782040066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,1024,0.005799110978841782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,2048,0.04056355688307021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,768,0.004820444517665439
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,1536,0.034127109580569796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,512,0.0038319999972979226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,256,0.002954666606254048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,1024,0.027523555689387854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,768,0.02437777817249298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,128,0.002291555619902081
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,512,0.020982222424613103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,64,0.001973333251145151
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,256,0.017047999633683097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,32,0.0021911110315057966
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,128,0.015617777903874716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,64,0.014466666513019137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,32,0.01440711153878106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,65536,3.1829314761691623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,65536,3.178473790486654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,12288,2.504880905151367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,10240,2.0778435601128473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,16384,3.2592595418294272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,10240,3.901152716742622
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,8192,1.6875555250379775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,7168,1.4855022430419922
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,12288,4.643185933430989
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,8192,3.103204515245226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,7168,2.6903875139024525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,6144,1.317855093214247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,6144,2.293347464667426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,5120,1.9053084055582683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,5120,1.108791987101237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,16384,6.28002209133572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,4096,0.93996885087755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,4096,1.482225735982259
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,3584,0.8409235742357043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,3584,1.2687288920084636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,2560,0.693303108215332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,2560,0.9199644724527994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,3072,0.7603884273105197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,3072,1.0396400027804904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,1536,0.5748364660474989
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,2048,0.6082888709174262
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,2048,0.7422560056050619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,1536,0.5336684650844997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,1024,0.44987646738688153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,768,0.3783084551493327
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,768,0.4093315601348877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,256,0.26783821317884654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,1024,0.4197919898562961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,512,0.33898401260375977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,512,0.37111200226677793
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,128,0.27640801005893284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,64,0.17548888259463838
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,32,0.16979911592271593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,128,0.2037982278399997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,64,0.2691404554578993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,256,0.3121235635545519
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,32,0.2611351013183594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,10240,0.61335113313463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,12288,0.7309048970540365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,16384,1.027226659986708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,12288,1.0329715940687392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,10240,0.900321748521593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,16384,1.4584017859564886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,8192,0.5075208875868056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,8192,0.711517333984375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,7168,0.47811465793185765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,7168,0.631530655754937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,6144,0.4527955585055881
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,6144,0.5471030871073405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,5120,0.4448213577270508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,4096,0.3471688959333632
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,3584,0.30907021628485787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,5120,0.3678559992048476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,3072,0.2830702198876275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,4096,0.30856532520718044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,2560,0.23871111869812012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,3584,0.27904976738823783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,2560,0.21211822827657065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,3072,0.24250133832295737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,2048,0.18358843856387666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,2048,0.1880160040325589
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,1024,0.09470666779412164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,1536,0.1577413347032335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,1536,0.1388888888888889
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,1024,0.12954666879442003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,768,0.0743466681904263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,768,0.11720089117685954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,256,0.03203377789921231
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,64,0.01722044414944119
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,512,0.10571022166146173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,128,0.08199555344051786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,256,0.08851199679904515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,128,0.02139555580086178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,32,0.014273777604103088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,512,0.05471377902560764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,64,0.079276442527771
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,32,0.07706222269270155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,16384,0.8700453440348307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,12288,1.0953013102213542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,16384,1.434409777323405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,65536,4.947185940212674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,12288,0.6333884663052028
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,10240,0.5442275471157497
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,10240,0.811357339223226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,8192,0.6484079890780979
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,8192,0.4292897648281521
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,7168,0.3949635558658176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,7168,0.6000035603841146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,6144,0.5275404188368056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,6144,0.32943643463982475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,65536,7.005374908447266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,5120,0.30094756020439994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,5120,0.43352799945407444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,4096,0.3523591094546848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,3584,0.3095182312859429
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,3584,0.23275823063320586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,4096,0.2527688874138726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,3072,0.2709999879201253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,3072,0.18765689267052543
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,2048,0.19133955902523467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,2560,0.23090489705403647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,1536,0.12556800577375624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,2048,0.14601155122121176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,2560,0.17099644078148735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,1536,0.1449928945965237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,1024,0.10216089089711507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,256,0.026055110825432673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,512,0.045647998650868736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,768,0.07780533366733126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,512,0.08150666952133179
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,128,0.016781333420011733
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,1024,0.09820977846781413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,64,0.014955555399258932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,768,0.09275644355350071
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,128,0.06341866652170818
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,32,0.011968889170222811
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,64,0.05860800213283963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,32,0.05878133243984646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,256,0.06816889180077447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,65536,4.429879082573785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,12288,0.5407928890652126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,65536,5.2167032029893665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,12288,0.7039110925462512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,10240,0.5728071000840929
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,16384,0.7329529126485189
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,16384,0.9224649005466037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,8192,0.4644835789998372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,8192,0.3820799986521403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,7168,0.4357724454667833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,7168,0.3255404366387261
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,10240,0.45463201734754777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,6144,0.3707137902577718
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,4096,0.2298791143629286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,4096,0.20751110712687174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,5120,0.2837422158983019
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,3584,0.2111039956410726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,3072,0.169159107738071
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,6144,0.3196400006612142
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,5120,0.2478853331671821
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,3072,0.16264532672034368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,3584,0.18447555436028373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,2560,0.14704355928632948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,1536,0.0897066659397549
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,2048,0.117411560482449
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,2048,0.12731556097666422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,2560,0.13741244210137263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,1536,0.10535999801423813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,1024,0.06379110945595635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,512,0.03579377796914842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,768,0.05001777741644117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,256,0.02299022177855174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,1024,0.08824266990025838
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,128,0.014486221803559197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,64,0.012698666916953193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,256,0.06192977560891045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,768,0.08035022020339966
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,512,0.07095111078686185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,32,0.009766221874290043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,64,0.052858667241202466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,128,0.05784533421198527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,32,0.053284443087048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,16384,0.7469813558790418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,12288,0.598290655348036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,12288,0.7499146461486816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,65536,3.957178751627604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,16384,0.8593662050035266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,10240,0.5593208736843532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,10240,0.505707581837972
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,65536,4.4507802327473955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,8192,0.42524443732367617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,7168,0.386408011118571
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,7168,0.3799022303687201
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,8192,0.42107378111945254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,5120,0.3006257745954725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,6144,0.3679200013478597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,6144,0.34071021609836155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,4096,0.22838134235805937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,4096,0.222799989912245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,3072,0.17358222272660997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,3584,0.20301332738664415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,5120,0.26945243941413033
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,3584,0.20144000318315294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,2560,0.14688177903493246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,2048,0.11873600218031143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,3072,0.20131999916500518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,2560,0.15661688645680746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,2048,0.13534043894873724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,1024,0.07000355588065253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,768,0.04011911153793335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,1536,0.1258960035112169
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,1024,0.0975946651564704
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,512,0.029020445214377508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,256,0.017281777328915067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,768,0.08896444241205852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,128,0.012578667038016848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,64,0.00997866690158844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,512,0.07783733473883735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,128,0.06833689080344306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,256,0.0746320022477044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,32,0.008344000412358178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,64,0.06634755267037286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,1536,0.08581866820653279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,32,0.06567377514309354
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,65536,3.5581758287217884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,12288,0.43993600209554035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,16384,0.6179199748569065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,65536,3.851898617214627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,12288,0.602950202094184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,10240,0.367340432273017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,16384,0.8364462322658963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,8192,0.32668622334798175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,7168,0.33670843972100156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,10240,0.5107119878133138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,6144,0.26806844605339897
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,7168,0.3768462340037028
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,5120,0.26712266604105633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,8192,0.41823021570841473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,6144,0.35028889444139266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,4096,0.23351822959052193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,3584,0.16294577386644152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,4096,0.17573422855801055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,5120,0.2828471130794949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,3584,0.19802399476369223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,3072,0.13438844680786133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,2560,0.12648267216152614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,1536,0.08551644616656834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,2048,0.1347279946009318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,2560,0.15730399555630153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,2048,0.12605422072940403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,3072,0.2021777762307061
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,1024,0.05010222064124214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,512,0.028911110427644517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,768,0.03918933206134372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,256,0.018203554881943595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,1024,0.09645244148042466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,128,0.012101333174440594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,64,0.00999022192425198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,1536,0.12580000029669866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,768,0.08531110816531712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,256,0.06787021954854329
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,512,0.07341333230336507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,128,0.06601955493291219
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,32,0.008425777984990014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,64,0.06304177973005506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,32,0.06229866875542534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,65536,3.31840451558431
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,12288,0.5060186915927464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,16384,0.6276293330722386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,12288,0.5944408840603298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,10240,0.41447822252909344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,16384,0.802469359503852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,8192,0.3433244493272569
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,65536,3.849501291910807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,10240,0.5019777615865072
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,8192,0.4125555621253119
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,6144,0.2799777719709608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,7168,0.3254106574588352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,7168,0.3713724348280165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,5120,0.2639075650109185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,6144,0.33760621812608504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,4096,0.20351378122965494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,5120,0.2740480105082194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,3584,0.17837599913279215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,4096,0.23212001058790419
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,3584,0.19753066698710123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,2560,0.12557600604163274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,3072,0.15078311496310765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,3072,0.2002862162060208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,2560,0.1510897742377387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,2048,0.1052631139755249
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,1024,0.051895999246173434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,2048,0.1306435532040066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,768,0.03903199897872077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,1536,0.10690489080217148
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,512,0.027990221977233887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,1536,0.0840195549858941
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,256,0.01683466633160909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,128,0.01182933317290412
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,1024,0.08826133277681138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,768,0.08114044533835517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,256,0.06381244129604764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,64,0.009611555271678502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,32,0.007791111038790808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,128,0.05807822280459934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,512,0.07354666789372762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,64,0.0572942230436537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,32,0.05400088760587904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,65536,2.8679936726888022
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,12288,0.3377048969268799
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,16384,0.45230309168497723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,16384,0.561590247684055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,10240,0.3003031147850884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,12288,0.4102799892425537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,10240,0.34682488441467285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,8192,0.23574577437506783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,6144,0.16685244772169325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,8192,0.3019804424709744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,7168,0.19266133838229707
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,5120,0.1430399947696262
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,7168,0.26345332463582355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,65536,3.8453246222602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,6144,0.2398106786939833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,4096,0.11374666955735947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,3584,0.10401155551274617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,5120,0.1876151164372762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,4096,0.14803377787272134
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,3584,0.12838399410247803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,2560,0.07503733370039198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,2048,0.059784001774258085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,3072,0.08602577447891235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,2560,0.09682133462693955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,1536,0.04644088943799337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,1024,0.032270222902297974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,3072,0.13727733823988172
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,768,0.02826933397187127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,1536,0.06458399693171184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,65536,2.235141330295139
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,512,0.023183999790085688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,2048,0.0806293355094062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,256,0.01349066694577535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,128,0.008472888833946651
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,1024,0.05327644281917148
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,64,0.0067528887755341
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,512,0.040056887600157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,768,0.04846488767200046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,256,0.03813955518934462
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,32,0.005967111223273807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,128,0.03479111194610596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,64,0.03271111183696323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,32,0.030379556947284277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,12288,0.2716791099972195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,16384,0.3628391159905328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,10240,0.22974843449062773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,12288,0.36895023451911074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,65536,2.971535152859158
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,8192,0.20321244663662383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,10240,0.3137413395775689
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,16384,0.522157351175944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,8192,0.24712978469000924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,6144,0.1763519975874159
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,7168,0.18531733089023164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,6144,0.17985688315497506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,5120,0.1392551130718655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,7168,0.2176720036400689
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,4096,0.11931555800967747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,4096,0.11131289270189072
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,5120,0.1352791123920017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,3584,0.09852977593739827
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,3072,0.08445244365268284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,2560,0.07266933388180204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,3584,0.09625244140625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,2048,0.058315555254618325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,1536,0.04455555478731791
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,1024,0.03398044572936164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,2560,0.07786489195293851
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,768,0.02681688964366913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,3072,0.09123288922839695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,2048,0.06796711020999484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,65536,1.521116468641493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,512,0.019187556372748483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,1536,0.05762666463851929
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,256,0.01204355557759603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,128,0.008068444828192392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,64,0.006211555666393704
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,768,0.044143110513687134
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,1024,0.04829511046409607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,512,0.03952177696757846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,32,0.0052888890107472735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,128,0.030602667066786025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,64,0.02906044324239095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,32,0.02865511178970337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,256,0.03637689020898607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,12288,0.24477778540717232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,16384,0.3061022228664822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,10240,0.2033440007103814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,12288,0.3711617787679036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,16384,0.5039306746588813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,10240,0.3132088979085286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,8192,0.167776001824273
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,7168,0.16442488299475774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,6144,0.15106666088104248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,8192,0.23933778868781197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,7168,0.21258666780259874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,5120,0.13490577538808188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,6144,0.1731182204352485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,4096,0.09942666689554851
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,65536,2.9919439951578775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,3584,0.08565777540206909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,5120,0.13354133235083684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,3584,0.09795555803510879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,3072,0.08755377928415935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,2560,0.06932355297936334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,4096,0.10989244116677178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,65536,1.6285342110527887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,2560,0.07618044482337104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,1536,0.04906488789452446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,3072,0.088191999329461
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,1024,0.03362755642996894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,2048,0.06722133027182685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,768,0.0270142224099901
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,2048,0.07459111346138848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,1536,0.06041333410474989
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,1024,0.04842488964398702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,256,0.011773332953453064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,768,0.043215112553702466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,512,0.01963200006220076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,128,0.008038221961922115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,512,0.0382008883688185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,64,0.006102222121424145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,32,0.005075555708673266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,64,0.02815822097990248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,128,0.02991644541422526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,32,0.02754755483733283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,256,0.03394666645261977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,12288,0.22884533140394422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,16384,0.3051599926418728
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,10240,0.1982640027999878
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,16384,0.5155102411905924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,12288,0.3777635627322727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,8192,0.15923378202650282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,10240,0.3078320026397705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,65536,2.984201855129666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,7168,0.16398221916622585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,8192,0.24503111839294434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,65536,1.3167698118421767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,6144,0.14514488644070095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,3584,0.0957546697722541
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,4096,0.13836444748772517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,5120,0.1321600013309055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,7168,0.21060710483127168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,4096,0.10831999778747559
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,3072,0.09837600257661606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,6144,0.17968355284796822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,5120,0.13090844949086508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,2560,0.07536177502738105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,3584,0.097034666273329
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,2048,0.060903999540540904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,1024,0.03384088807635837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,1536,0.04881333311398824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,2048,0.0639351142777337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,1536,0.05371555685997009
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,768,0.026611554953787062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,512,0.0191600008143319
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,2560,0.07279821899202135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,256,0.011647111011876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,1024,0.05080088973045349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,768,0.03979377614127265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,128,0.008008889026112026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,3072,0.0891759991645813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,512,0.034613334470325045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,32,0.004608889006906086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,128,0.027067555321587458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,64,0.0064284445510970215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,256,0.02979644470744663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,32,0.025440000825458105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,64,0.0247697780529658
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,12288,0.1844560040367974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,16384,0.25822220908270943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,10240,0.149472885661655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,8192,0.1358213292227851
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,12288,0.2777626779344347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,16384,0.39847999148898655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,7168,0.11246755388047959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,10240,0.22805510626898873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,8192,0.16878933376736113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,6144,0.09136799971262614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,4096,0.06218577755822075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,7168,0.14132355319129095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,5120,0.07699466413921781
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,6144,0.122524446911282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,3584,0.052277333206600614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,4096,0.06961866882112291
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,3072,0.047967112726635404
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,65536,1.0346044964260526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,5120,0.08740888701544867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,3584,0.062000890572865806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,65536,2.989659627278646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,2560,0.03938577903641595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,2048,0.035122666094038225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,1536,0.026798221800062392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,3072,0.05581421984566582
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,2560,0.052279111411836415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,1024,0.02064533366097344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,2048,0.044532444741990834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,768,0.01573422220018175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,1536,0.038703110482957624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,256,0.007598222129874759
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,1024,0.03262133399645487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,768,0.02942133280966017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,128,0.00563466673096021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,512,0.026762665973769292
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,512,0.011626666618718041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,64,0.004545777622196409
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,256,0.02307644486427307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,32,0.003903111235962974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,128,0.020596444606781006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,64,0.01978222197956509
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,32,0.02036266691154904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,12288,0.16333156161838108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,16384,0.19308000140719941
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,10240,0.14121154944101968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,16384,0.3887786600324843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,12288,0.2582168844011095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,8192,0.1204951074388292
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,10240,0.20892800225151908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,7168,0.09975644614961411
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,65536,0.814452436235216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,7168,0.1269893381330702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,8192,0.14818310737609863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,6144,0.08561333020528157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,6144,0.09883199797736274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,4096,0.058542225095960826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,5120,0.07967288626564874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,5120,0.08602933088938396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,3584,0.051669332716200084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,4096,0.06541777981652154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,3584,0.05938933293024699
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,3072,0.04951644606060452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,2560,0.03801244497299194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,65536,2.5053350660536027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,3072,0.05618933174345228
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,2560,0.04764088988304138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,2048,0.04440977838304308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,1024,0.018516444497638278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,1536,0.02625688910484314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,768,0.015430221954981485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,512,0.010856000085671743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,1536,0.036892443895339966
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,2048,0.03120533294147915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,256,0.007170666423108842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,768,0.028456889920764502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,128,0.005295111073387994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,1024,0.030271109607484605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,64,0.004459555364317364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,32,0.0037324445115195382
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,256,0.021557332740889654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,512,0.025395555628670588
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,128,0.018974221414989896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,64,0.018061333232455783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,32,0.018258665998776753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,12288,0.11760444111294216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,16384,0.1498311095767551
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,10240,0.1006586684121026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,12288,0.23699466387430826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,8192,0.0818471113840739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,16384,0.3704675568474664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,65536,0.5818630854288737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,10240,0.18668977419535318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,7168,0.0907555553648207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,8192,0.13388800621032715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,6144,0.09210222297244602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,5120,0.06358577807744344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,7168,0.11069955428441365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,65536,2.4791013929578996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,6144,0.09141155746248032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,5120,0.07689333624309964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,4096,0.05635822481579251
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,2560,0.039946665366490684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,3072,0.04011022382312351
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,3584,0.045422222879197865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,4096,0.06428355640835233
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,3584,0.05894933144251505
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,3072,0.052793777651256986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,2048,0.02999911043379042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,1536,0.025814221964942083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,1024,0.018008889423476327
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,2560,0.049207998646630176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,768,0.014616888430383472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,2048,0.04141244292259216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,512,0.010737777584128909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,1536,0.03507733345031738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,1024,0.029256890217463177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,256,0.0070053330726093715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,768,0.026846221751636926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,64,0.004320000194840961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,128,0.00518488883972168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,32,0.0031226666437255014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,512,0.02443644404411316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,256,0.020225778222084045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,64,0.01744888888465034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,128,0.01808888879087236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,32,0.016899555921554565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,12288,0.08104533619350858
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,16384,0.09585866663191055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,10240,0.06487377484639485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,12288,0.2171928882598877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,8192,0.05228711167971293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,65536,0.43646399180094403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,16384,0.35849777857462567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,7168,0.04704622096485562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,10240,0.16774488819970024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,6144,0.043028444051742554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,8192,0.119595554139879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,5120,0.03635199864705404
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,4096,0.03101244568824768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,7168,0.10369422038396199
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,3584,0.030067556434207495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,5120,0.07352888584136963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,6144,0.08708533313539292
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,3072,0.026388444834285315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,4096,0.061942219734191895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,2560,0.022480888499153986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,2048,0.01774488886197408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,3584,0.05673066775004069
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,3072,0.050048887729644775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,1536,0.014462222655614218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,2560,0.04440177811516655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,1024,0.01053155544731352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,2048,0.03847022189034356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,768,0.008752889103359645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,1536,0.03305600086847941
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,65536,2.4636720021565757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,512,0.006735999965005451
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,256,0.005173333403136996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,128,0.004003555410438114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,1024,0.026904000176323786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,768,0.024117334021462336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,64,0.0032595555401510666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,512,0.021153777837753296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,32,0.003085333233078321
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,128,0.01623911162217458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,64,0.015096889601813423
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,256,0.018020444446139865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,32,0.014815110299322339
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,12288,0.06949155860477023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,16384,0.08128533098432753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,10240,0.05043466554747688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,65536,0.35508354504903156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,12288,0.2073964410358005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,8192,0.04154755671819051
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,10240,0.15798844231499568
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,7168,0.036526223023732506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,6144,0.03391644358634949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,16384,0.3334231111738417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,8192,0.12298756175571018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,7168,0.09836088948779637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,5120,0.03001333276430766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,4096,0.026907554931110803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,3584,0.023201778531074524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,6144,0.08505333132214016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,3072,0.020605333977275424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,5120,0.07319555679957072
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,4096,0.0612551106346978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,2560,0.018929777873886954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,3584,0.05660533242755466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,2048,0.01676177812947167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,3072,0.04939644535382589
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,1536,0.01384800010257297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,1024,0.010016889207892949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,2048,0.03826399975352817
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,768,0.008387555678685507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,2560,0.043840001026789345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,1536,0.032130665249294706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,512,0.00664444433318244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,1024,0.02658133374320136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,256,0.005005333158704969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,128,0.0037742224004533556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,64,0.00299466670387321
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,768,0.023774221539497375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,512,0.0203484445810318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,32,0.0028115556471877625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,128,0.015375110838148328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,65536,2.4723421732584634
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,64,0.014480888843536377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,32,0.0144177774588267
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,12288,0.08357510964075725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,16384,0.10434755351808335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,256,0.017218665944205392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,65536,0.27498666445414227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,10240,0.03391644358634949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,12288,0.20173866218990752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,7168,0.02665333284272088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,8192,0.028411553965674505
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,6144,0.023902222514152527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,10240,0.145304004351298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,5120,0.02075111038155026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,16384,0.32694845729404026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,8192,0.11186310980055068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,4096,0.018192888961897958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,7168,0.09802310996585423
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,6144,0.08432977729373509
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,3584,0.017312889297803242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,5120,0.07277866866853502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,3072,0.017323555217848886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,2560,0.01570044457912445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,4096,0.060863110754224986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,3584,0.055339554945627846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,2048,0.013774221969975365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,3072,0.049357334772745766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,2560,0.043439110120137535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,1536,0.010509333676762052
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,1024,0.007699555820888943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,768,0.006487999939256244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,2048,0.03763555486996969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,1536,0.03169066707293192
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,512,0.005381333331267039
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,1024,0.02610133257177141
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,768,0.023208889696333144
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,256,0.003951111187537511
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,128,0.0033039999090962936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,64,0.0026435556097163092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,256,0.016884444488419425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,512,0.020057777563730877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,32,0.002509333400262727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,64,0.014451555079884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,128,0.015209777487648858
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,65536,2.4713832007514105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,16384,0.02650044361750285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,32,0.014268444644080268
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,10240,0.01934400035275353
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,65536,0.22132978174421522
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,12288,0.0221742226017846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,8192,0.016711999972661335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,7168,0.01630577776167128
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,10240,0.13823378086090088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,12288,0.19097510973612467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,16384,0.31595643361409503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,8192,0.10985955264833237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,6144,0.016559110747443307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,5120,0.014097778333557976
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,4096,0.012031111452314587
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,7168,0.09715377622180515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,3584,0.011552000211344825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,6144,0.08457066615422566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,3072,0.010610666539933948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,5120,0.0728071133295695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,2560,0.009550222092204625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,4096,0.0612551106346978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,3584,0.05535733368661669
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,2048,0.00981244444847107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,1536,0.008217777642938826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,3072,0.049518220954471164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,2560,0.04346488912900289
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,1024,0.006360888895061281
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,2048,0.037913779417673744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,1536,0.0321066677570343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,768,0.005577777822812398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,512,0.004578666554556953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,256,0.003592888928121991
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,1024,0.026351110802756414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,768,0.023306666149033442
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,128,0.0028986665937635633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,64,0.0022960000981887183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,512,0.02033866610791948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,256,0.01702133317788442
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,32,0.0021671110557185277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,128,0.015587555037604438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,64,0.014398222168286642
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,16384,0.019543111324310303
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,32,0.014451555079884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,12288,0.016160888804329764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,65536,2.480708440144857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,65536,0.18377510706583658
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,10240,0.016506666938463848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,8192,0.013849777479966482
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,7168,0.012725333372751871
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,12288,0.18104267120361328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,16384,0.30736621220906574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,8192,0.1096186637878418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,6144,0.011703111231327057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,10240,0.13448354932996961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,5120,0.01036800030204985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,7168,0.09694844484329224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,3584,0.008310221963458592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,6144,0.08425955639945136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,5120,0.0728986660639445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,4096,0.06121777825885349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,4096,0.00871377769443724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,3072,0.007735999921957652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,2560,0.0070364442136552595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,3584,0.05530222256978353
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,3072,0.049477332168155246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,1536,0.009094222552246517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,2048,0.037680887513690524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,1024,0.006643555644485686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,2048,0.0064426664676931165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,768,0.005492444253630108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,512,0.004286222159862518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,1024,0.02621777852376302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,1536,0.0318177772892846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,768,0.023331556055280898
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,2560,0.04353511002328661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,256,0.003140444349911478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,128,0.002489777695801523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,64,0.0020906666500700847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,512,0.02020088831583659
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,32,0.0022862222459581164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,256,0.01682222220632765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,128,0.015426667200194465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,64,0.014564444621404013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,16384,0.025400888588693406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,32,0.014246222045686511
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,65536,2.4848249223497176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,12288,0.02095288866096073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,65536,0.1402204434076945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,10240,0.017472889688279893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,8192,0.014579556054539151
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,7168,0.014127999544143677
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,10240,0.13476978407965765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,12288,0.18188177214728463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,16384,0.3118977811601427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,6144,0.012531555361217923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,8192,0.10985510879092747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,5120,0.010839111275143094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,7168,0.09674933221605088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,4096,0.009348444640636444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,3584,0.009139555195967356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,6144,0.08452266454696655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,5120,0.07279022534688313
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,3072,0.008399999803966945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,4096,0.0610719985432095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,3584,0.05493866735034519
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,2560,0.00759733302725686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,2048,0.006812444577614467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,1536,0.008805333740181392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,1024,0.0064479998416370815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,3072,0.04937511020236545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,2560,0.04354133208592733
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,2048,0.03758044375313653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,768,0.005295111073387994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,1536,0.031763556930753924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,512,0.004126222183307012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,256,0.002975111206372579
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,1024,0.02612177696492937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,128,0.0023448889454205832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,768,0.023186667097939387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,512,0.020066666934225295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,64,0.0019786666250891155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,256,0.016594666573736403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,32,0.0021884443445338146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,128,0.01536266671286689
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,64,0.014535110857751636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,65536,2.4880426194932723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,32,0.014299554957283868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,16384,0.01462222304609087
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,65536,0.09668888648351033
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,10240,0.010936000280910067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,12288,0.012092444631788464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,8192,0.0094035557574696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,7168,0.010415111151006486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,10240,0.13472533226013184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,12288,0.1790675587124295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,8192,0.10954578055275811
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,6144,0.007128000259399414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,5120,0.006632888896597757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,7168,0.0964417788717482
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,16384,0.30656178792317706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,4096,0.007374222079912822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,3584,0.0069857777820693115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,6144,0.08429866366916233
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,5120,0.07252089182535808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,3072,0.006461333483457565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,4096,0.06096000141567654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,2560,0.005842666659090254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,3584,0.05491022268931071
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,2048,0.005504888792832692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,1536,0.0076044441925154785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,3072,0.04927822285228305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,2560,0.043474667602115206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,1024,0.00566133318675889
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,2048,0.03754577702946133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,512,0.0037617778612507712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,768,0.0047484445903036325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,1536,0.03170844581392076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,1024,0.02609688871436649
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,768,0.02288977801799774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,256,0.0028880000528362063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,128,0.0022408889813555614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,64,0.002179555594921112
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,32,0.0021760000122918025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,512,0.019916445016860962
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,256,0.016826666063732572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,128,0.015087111128701104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,64,0.014562666416168213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,32,0.013990221752060784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,65536,2.4892186058892145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,65536,2.497049755520291
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,10240,1.4654293060302734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,12288,1.7587270736694336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,16384,2.5283724466959634
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,8192,2.415330674913194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,8192,1.1856418185763888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,10240,2.939294179280599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,12288,3.7607633802625866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,7168,1.0909546746148002
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,7168,2.074728012084961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,6144,0.9594986173841688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,16384,4.5648418002658415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,5120,0.7886568705240885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,4096,0.6527182261149088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,5120,1.5349359512329102
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,6144,1.842641830444336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,3584,0.9639075597127279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,4096,1.0966542561848958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,3584,0.5963990953233507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,2560,0.4729608959621853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,2048,0.5677093399895562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,2560,0.7046657668219672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,1536,0.3473697768317328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,1536,0.44717152913411456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,2048,0.4261315663655599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,3072,0.5483511288960775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,3072,0.856728023952908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,1024,0.35086666213141543
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,256,0.17021777894761828
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,512,0.2363679938846164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,128,0.06655110915501912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,256,0.23006044493781197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,768,0.3037475479973687
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,768,0.26156266530354816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,32,0.033490667740503945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,128,0.20213599999745688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,1024,0.2939857906765408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,64,0.04793599910206265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,32,0.18181688255733916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,512,0.23633154233296713
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,64,0.18150577280256483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,12288,0.5320631133185493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,10240,0.7392480108473035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,16384,0.8146693441602918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,10240,0.46679019927978516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,12288,0.8844435479905871
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,16384,1.1502159966362846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,8192,0.5659013324313694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,8192,0.38595999611748594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,6144,0.42224621772766113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,7168,0.3333173327975803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,6144,0.29938578605651855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,7168,0.5199795299106175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,5120,0.34814755121866864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,4096,0.1903484397464328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,4096,0.28659555647108287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,3584,0.24245956208970812
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,3072,0.21198399861653647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,5120,0.25801067882113987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,3584,0.1897653341293335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,2048,0.13148266739315456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,3072,0.16223288906945124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,2048,0.12116000387403701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,1024,0.06747911373774211
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,1536,0.10263644324408637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,2560,0.190193772315979
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,768,0.05317155520121256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,2560,0.14759555127885607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,512,0.03956088754865859
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,1024,0.08881066905127631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,256,0.022688888841205176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,768,0.08185599909888373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,128,0.01500888831085629
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,512,0.07286133368810017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,1536,0.10614399777518378
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,32,0.010296888649463654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,64,0.013385777672131857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,128,0.058223108450571694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,64,0.054261333412594266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,32,0.0543066660563151
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,256,0.06452800167931451
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,12288,0.6270782152811686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,65536,3.508728875054253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,12288,0.438573333952162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,16384,0.6042070918613011
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,16384,0.8024666574266223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,8192,0.28921688927544487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,8192,0.3752302328745524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,10240,0.3708871205647786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,10240,0.4968329005771213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,7168,0.47057776980929905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,7168,0.25285598966810435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,6144,0.2153680059644911
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,6144,0.4074062241448297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,5120,0.19636089271969268
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,5120,0.3437537882063124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,4096,0.15058933364020452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,3584,0.2318577766418457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,3072,0.19938400056627062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,4096,0.26634756724039715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,3584,0.13273599412706164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,3072,0.12168267038133408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,2560,0.1550782256656223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,2048,0.12712977992163763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,65536,5.362736172146267
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,2560,0.11101333300272624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,1024,0.06387911240259807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,2048,0.0945884452925788
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,768,0.04599377844068739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,1536,0.08277599679099189
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,1024,0.07048177719116211
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,768,0.06410400072733562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,256,0.017588444881969027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,1536,0.10126844379636978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,128,0.012553777959611682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,256,0.05101066827774048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,64,0.010441777606805166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,512,0.030155556069480047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,512,0.057686222924126514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,32,0.008866666919655269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,128,0.04616444309552511
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,32,0.04407822092374166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,65536,2.9541422526041665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,64,0.04412000046836006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,16384,0.7572008768717448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,16384,0.6225013203091091
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,12288,0.44749244054158527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,65536,3.9818988376193576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,12288,0.5791591008504232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,7168,0.30174756050109863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,8192,0.2976035541958279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,10240,0.46341689427693683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,8192,0.3802693419986301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,10240,0.3808142344156901
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,6144,0.2657537725236681
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,6144,0.21292711628807914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,5120,0.23976710107591417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,5120,0.18183555867936876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,4096,0.1425271166695489
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,7168,0.25845689243740505
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,4096,0.19508888986375597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,3584,0.16631466812557644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,3584,0.131151106622484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,3072,0.11809955702887641
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,2560,0.16405155923631456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,3072,0.15040711561838785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,2048,0.0911137792799208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,2560,0.10745155811309814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,65536,2.7096996307373047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,2048,0.1283457809024387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,1536,0.09013955460654365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,1024,0.05760711431503296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,512,0.02858933475282457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,256,0.018211555149820115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,1536,0.07972444428337945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,768,0.06748621993594699
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,768,0.042746666404936046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,512,0.053192890352673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,1024,0.06616799698935615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,128,0.01273333364062839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,65536,2.922815958658854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,32,0.04035733474625482
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,32,0.008416888614495596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,128,0.04258666767014397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,64,0.01092711091041565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,256,0.04736977815628052
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,64,0.04105510976579454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,10240,0.3265999952952067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,12288,0.425784879260593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,16384,0.5319857597351074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,12288,0.41316888067457413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,16384,0.5116222169664171
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,10240,0.3557164404127333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,8192,0.2650711006588406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,7168,0.2594737741682264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,8192,0.27541422843933105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,6144,0.21395111083984375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,5120,0.15767555766635472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,5120,0.17326754993862578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,6144,0.21546667151980928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,7168,0.24784710672166613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,4096,0.13147644201914468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,3072,0.096961776415507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,3584,0.10973244243197972
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,3584,0.12651911046769884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,2560,0.07909599939982097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,4096,0.14286755190955266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,3072,0.11349066098531087
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,2048,0.06153866979810926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,1536,0.04878488845295376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,2560,0.11230044894748265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,768,0.025342222717073228
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,1024,0.03709422217475043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,2048,0.08378399742974176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,1536,0.0700133310423957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,1024,0.058555556668175586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,256,0.01262399968173769
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,512,0.020032000210550096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,768,0.051445331838395864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,256,0.04212177793184916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,512,0.049475553962919444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,64,0.03615999884075589
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,128,0.04131733377774557
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,128,0.008685333033402761
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,64,0.0069004446268081665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,65536,2.325568093193902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,32,0.006127111199829314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,32,0.035976890060636736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,12288,0.39510220951504177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,65536,2.3478177388509116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,16384,0.5055688752068414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,12288,0.3557608922322591
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,8192,0.2612568802303738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,10240,0.3187013202243381
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,16384,0.49801333745320636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,8192,0.256677336162991
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,10240,0.3040835592481825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,7168,0.2159555620617337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,6144,0.18449778027004668
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,5120,0.1407795614666409
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,7168,0.21463467015160453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,6144,0.1771164470248752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,4096,0.11661067273881699
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,3584,0.10096089045206706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,3584,0.10480711195203994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,3072,0.0879439976480272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,5120,0.1521084441079034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,2560,0.07411555449167888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,2048,0.06014577547709147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,3072,0.09172444211112128
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,4096,0.12130844593048096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,65536,1.945831086900499
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,1536,0.046430223517947726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,2560,0.09575821955998738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,1024,0.03398933344417148
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,768,0.02496355606449975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,512,0.019262222780121695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,1024,0.05293600095642937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,768,0.041991998751958214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,1536,0.056157330671946205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,256,0.012679110798570844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,128,0.0084906667470932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,256,0.03511111272705926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,64,0.006792888873153263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,2048,0.0692088868882921
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,512,0.038915554682413735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,128,0.030989332331551447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,65536,2.276156531439887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,64,0.028894222444958154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,32,0.006135111053784688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,32,0.02949777907795376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,16384,0.39375999238755965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,12288,0.30105333858066136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,10240,0.32507289780510795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,12288,0.34630666838751895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,16384,0.45213953653971356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,8192,0.25719467798868817
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,10240,0.29400089051988393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,7168,0.19652088483174643
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,8192,0.2148346636030409
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,7168,0.18500978416866729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,5120,0.1255440049701267
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,6144,0.16489333576626247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,5120,0.1394933329688178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,4096,0.11220622062683105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,4096,0.10405511326260036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,3584,0.09882666667302449
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,2560,0.07377066877153185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,3584,0.091011557314131
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,3072,0.08822400040096706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,3072,0.08591910865571763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,2560,0.07396444347169664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,2048,0.05921866496404012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,1536,0.04552177919281853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,1024,0.034006221426857844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,6144,0.15980354944864908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,2048,0.05983466572231717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,1536,0.05189866820971171
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,1024,0.04423911041683621
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,65536,1.6290773815578883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,768,0.03902755512131585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,256,0.012103999654452005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,768,0.02715377840730879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,128,0.009232000344329411
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,512,0.019235556324323017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,64,0.006232000059551663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,32,0.006124444305896759
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,256,0.031186666753556993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,128,0.027987556325064764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,32,0.026615111364258662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,64,0.027091556125217017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,512,0.03475733266936408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,10240,0.2084479994244046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,16384,0.3453982141282823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,12288,0.2535511122809516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,12288,0.30713333023919
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,65536,2.244767930772569
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,16384,0.4226240052117242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,8192,0.17778578069474962
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,10240,0.2563413249121772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,7168,0.1604506704542372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,8192,0.19224711259206137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,7168,0.17550933361053467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,6144,0.14668799771202937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,6144,0.14230222172207302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,4096,0.1018035544289483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,5120,0.11388799879286025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,3584,0.09795822037590875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,4096,0.09358666340510051
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,5120,0.12760266992780897
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,3584,0.08379822307162815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,2560,0.07028533352745904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,3072,0.0795324444770813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,2560,0.06765155659781562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,2048,0.06268977456622653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,3072,0.0805457764201694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,1536,0.05040977729691399
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,1024,0.03340177734692892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,768,0.027287999788920086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,65536,1.532678180270725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,1536,0.05031377739376492
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,512,0.01939288940694597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,1024,0.04214310977194044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,128,0.008020444048775567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,768,0.03761333227157593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,256,0.011696889168686338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,512,0.037965334124035306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,64,0.006131555471155379
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,256,0.029339555237028334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,32,0.005223999834722943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,64,0.025791111919615004
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,128,0.02716444432735443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,32,0.025361778007613286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,2048,0.05914755662282308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,12288,0.19481778144836426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,16384,0.2571360005272759
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,16384,0.30598399374220103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,10240,0.15825511349572077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,12288,0.24145600530836317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,65536,2.243115531073676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,8192,0.12165510654449463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,10240,0.18479732672373453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,7168,0.10015288988749187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,7168,0.10677599906921387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,5120,0.07708533604939778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,8192,0.13635378413730198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,4096,0.06920178069008721
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,65536,1.145250638326009
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,6144,0.0882915523317125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,6144,0.08725955751207139
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,5120,0.0796746677822537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,3584,0.055474665429857045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,4096,0.06226666768391927
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,3072,0.045351111226611666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,3584,0.06457955307430692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,2048,0.031982223192850746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,3072,0.0525182220670912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,1536,0.030402666992611353
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,2560,0.04279911186960009
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,1024,0.01872711049185859
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,2560,0.04654399885071648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,768,0.01553600033124288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,1536,0.036728888750076294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,512,0.011252444651391772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,2048,0.04125510983996921
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,256,0.0074551113777690465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,768,0.02977333466211955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,64,0.00462400002612008
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,128,0.005506666584147348
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,512,0.02644088864326477
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,256,0.022827555735905964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,32,0.004010666575696734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,128,0.021316443880399067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,64,0.020377778344684176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,32,0.019648889700571697
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,1024,0.03270311156908671
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,16384,0.2243724399142795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,12288,0.17134577698177764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,65536,1.8685991499159071
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,12288,0.2116995520061917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,16384,0.2855493227640788
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,10240,0.155894226498074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,8192,0.11281243960062663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,7168,0.09788888692855835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,6144,0.09102044502894084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,8192,0.11337688234117295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,7168,0.09866044256422256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,5120,0.07225333319769965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,4096,0.061556445227728955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,10240,0.16548177931043836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,6144,0.08245155546400282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,3584,0.05146577623155382
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,5120,0.07265866465038724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,3072,0.04617955618434482
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,4096,0.060784889592064746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,3072,0.05086133215162489
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,3584,0.055890666113959424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,65536,1.124659538269043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,2048,0.033506668276256986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,1536,0.02624977793958452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,2560,0.04579200016127693
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,1536,0.03537688983811273
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,768,0.014896000425020853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,2560,0.038219554556740656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,1024,0.019256888164414298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,2048,0.040629333919949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,512,0.01090844472249349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,1024,0.03049066662788391
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,256,0.007585778004593319
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,128,0.005427555491526921
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,768,0.027658666173617046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,512,0.02480711042881012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,256,0.02221333318286472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,64,0.0047111109726958806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,32,0.0038862224254343244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,128,0.019742222295867074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,64,0.01923022170861562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,32,0.01864355636967553
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,16384,0.21056711673736572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,12288,0.15557066599527994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,16384,0.2697937753465441
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,65536,1.8086925082736547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,10240,0.13069155481126574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,12288,0.19187377558814156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,10240,0.15004089143541124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,7168,0.09990400075912476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,8192,0.11077244414223565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,8192,0.10540888706843059
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,5120,0.07153244151009454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,6144,0.08415289057625665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,7168,0.09134488635592991
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,4096,0.057929780748155385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,6144,0.07975111405054729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,65536,0.8636959923638238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,5120,0.06969688998328315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,3584,0.05125688844256931
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,3072,0.04555022385385302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,2048,0.03286400106218126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,2560,0.03804711169666714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,1536,0.026143110460705225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,4096,0.05864622195561727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,3584,0.05428622166315714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,1024,0.01835466590192583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,2560,0.04422933194372389
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,2048,0.0388426681359609
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,3072,0.052390221092436046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,768,0.014836443795098199
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,512,0.010952889091438718
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,1536,0.03444622291458978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,256,0.007224000162548489
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,128,0.0053182223604785065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,1024,0.028852442900339764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,768,0.026491555902693007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,64,0.004429333325889376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,32,0.003765333443880081
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,256,0.020607110526826646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,128,0.018737778067588806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,512,0.0241333329015308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,64,0.018008000320858426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,32,0.0181013329161538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,16384,0.16218133767445883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,12288,0.11751466327243382
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,65536,1.7798692915174696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,10240,0.10139466656578912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,12288,0.17600088649325898
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,16384,0.2567439874013265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,10240,0.13462755415174696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,8192,0.08500799867841934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,7168,0.09698222080866496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,6144,0.07851644357045491
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,5120,0.07131555345323351
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,65536,0.7318053245544434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,7168,0.08937866820229425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,6144,0.07890044318305121
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,8192,0.10147377517488267
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,4096,0.05380800035264757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,3072,0.04160088962978787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,3584,0.04819733235571119
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,4096,0.06294577651553683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,3584,0.053655998574362866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,2560,0.0369759996732076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,2048,0.032758222685919866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,1536,0.02606933315594991
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,5120,0.06853333446714613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,2560,0.04342577854792277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,2048,0.03967644439803229
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,3072,0.048617776897218495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,768,0.014731556177139282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,1024,0.01819022165404426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,1536,0.03355022271474203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,256,0.007111999723646376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,1024,0.028396444188223943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,512,0.010987555815113915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,64,0.004354666504595014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,512,0.022963555322753057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,128,0.0052533331844541765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,256,0.020044444335831534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,32,0.0033128888656695685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,128,0.018177777528762817
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,64,0.017838221457269456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,32,0.017462222112549674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,768,0.02591733303334978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,16384,0.13007377253638372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,12288,0.09436444441477458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,10240,0.0847724477450053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,65536,1.7615813149346247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,16384,0.2423093318939209
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,12288,0.16746932930416533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,8192,0.06492977672153048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,10240,0.12508977784050837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,7168,0.060227553049723305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,65536,0.6018222173055013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,6144,0.050739556550979614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,8192,0.09827999936209784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,5120,0.04398933384153578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,4096,0.037414222955703735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,7168,0.08709333340326945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,6144,0.07688622342215644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,5120,0.07154311074150933
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,3072,0.032109333409203425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,4096,0.056736886501312256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,3584,0.05188266767395867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,3584,0.032791998651292585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,2560,0.027767111857732136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,3072,0.04684177703327603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,1536,0.017248888810475666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,2048,0.021554667088720534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,2560,0.04163555635346307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,1024,0.012523555921183692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,768,0.010274666878912184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,2048,0.03669688767857022
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,512,0.0076453329788313965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,1024,0.026762665973769292
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,256,0.005326222214433882
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,128,0.004267555558019214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,768,0.024120888776249353
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,512,0.021359999974568684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,256,0.01864888932969835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,64,0.003548444559176763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,1536,0.0322693321439955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,128,0.01683999929163191
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,32,0.002918222298224767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,64,0.015997333659066092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,32,0.015740444262822468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,16384,0.10146666897667779
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,12288,0.07536355654398601
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,10240,0.06670400169160631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,16384,0.2259404394361708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,8192,0.055939555168151855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,12288,0.15004444122314453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,7168,0.049040890402264066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,65536,0.48056888580322266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,6144,0.044159111049440175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,65536,1.7472675111558702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,10240,0.12227288881937663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,8192,0.09625333547592163
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,7168,0.08585511313544379
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,5120,0.03691822290420532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,6144,0.07599466376834445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,4096,0.033703111939960055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,3584,0.029637333419587877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,4096,0.055139554871453174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,2560,0.022654222117529974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,3584,0.05058311091528999
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,3072,0.045401778486039907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,2048,0.01776088939772712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,3072,0.026986666851573523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,1536,0.014634667171372307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,2560,0.04026222229003906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,5120,0.06557155317730375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,768,0.008896888958083259
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,2048,0.035498665438758
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,1024,0.010568888651000129
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,1536,0.03050133254792955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,256,0.005044444567627377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,1024,0.025278222229745653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,768,0.022784888744354248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,128,0.003961777935425441
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,512,0.020210666788948905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,64,0.0034471110751231513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,256,0.01742577718363868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,512,0.006775110960006714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,128,0.015645333462291293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,32,0.003080000066094928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,64,0.015009777413474189
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,32,0.014903111590279473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,16384,0.06404978036880493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,12288,0.05141422152519226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,10240,0.04783911175198025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,8192,0.034705777963002525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,16384,0.2152871158387926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,7168,0.030096888542175293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,12288,0.14279022481706408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,65536,0.354860438240899
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,10240,0.1192657815085517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,8192,0.09482489029566447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,6144,0.02680088910791609
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,5120,0.024315555890401203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,4096,0.020451555649439495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,65536,1.7440542644924586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,6144,0.0747013356950548
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,5120,0.06475289000405206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,4096,0.05482133229573568
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,3584,0.049572444624371
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,3072,0.016925333274735343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,3584,0.01903199983967675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,2560,0.01550222271018558
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,1536,0.012108444339699216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,7168,0.08516977892981635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,3072,0.04473511046833462
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,2560,0.039502223332722984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,2048,0.01572444372706943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,2048,0.03475111060672336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,768,0.007535999847782983
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,512,0.006048000107208888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,1024,0.008985777695973715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,1024,0.024525332782003615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,256,0.0045422220395671
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,128,0.003624000069167879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,768,0.02228533393806881
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,512,0.019113777412308585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,256,0.016785777277416654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,64,0.0029279999434947968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,1536,0.029814223448435467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,32,0.002712888850106133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,128,0.015029332704014249
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,64,0.014364444547229342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,32,0.014097778333557976
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,16384,0.10672533512115479
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,12288,0.03988444474008348
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,10240,0.036280890305837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,65536,0.2758879926469591
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,8192,0.02900000082121955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,7168,0.026771555344263714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,12288,0.1399635606341892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,16384,0.20722666051652697
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,10240,0.11699288421207005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,6144,0.024108444650967915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,5120,0.020918221937285528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,8192,0.09482489029566447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,7168,0.08455200327767266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,65536,1.7505022684733074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,6144,0.07471111085679796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,4096,0.018031999468803406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,3584,0.017337777548366122
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,5120,0.06459555361005995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,3072,0.017553778158293832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,4096,0.0544417765405443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,2560,0.01572355628013611
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,2048,0.01386666629049513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,1536,0.01052177780204349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,3584,0.04943733414014181
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,3072,0.04485866758558485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,1024,0.007771555748250749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,2560,0.03941333293914795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,768,0.006510222123728857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,512,0.005387555393907759
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,1536,0.029636445972654555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,1024,0.024311999479929607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,2048,0.034437331888410784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,768,0.02181511123975118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,256,0.004064889003833135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,128,0.0034142221427626083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,512,0.019125334090656705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,64,0.0026604444202449587
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,32,0.002471999989615546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,128,0.01498311095767551
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,64,0.014242667290899487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,32,0.0141004439857271
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,16384,0.033992889854643084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,256,0.016543999314308167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,12288,0.02648977769745721
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,65536,0.22262220912509492
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,10240,0.024028443627887305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,8192,0.02036711076895396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,16384,0.19682844479878744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,7168,0.0188773340649075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,12288,0.138629330529107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,6144,0.017272000511487324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,10240,0.11568800608317058
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,8192,0.09468444188435872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,5120,0.01704266667366028
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,4096,0.014268444644080268
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,7168,0.08476266596052383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,65536,1.7641226450602214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,3584,0.013655111193656921
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,3072,0.012431999875439538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,2560,0.01124888906876246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,5120,0.0645973351266649
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,6144,0.0748622218767802
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,3584,0.049674666590160794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,2048,0.009887110855844285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,4096,0.05448977814780342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,1024,0.006375999914275275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,1536,0.008356444537639618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,2560,0.039480888181262545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,2048,0.03432622220781114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,3072,0.044701334502961904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,768,0.005568000177542369
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,512,0.004571555389298333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,1536,0.029633777009116277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,1024,0.0244186669588089
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,256,0.0037066667444176147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,128,0.003048000029391713
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,64,0.0024862223201327855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,768,0.021909332937664453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,512,0.018975999620225694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,32,0.0022799999763568244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,256,0.016388444436921012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,128,0.015031110909250049
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,16384,0.023730667101012334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,64,0.014319999350441827
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,32,0.014157333307796054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,12288,0.019152889649073284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,10240,0.016979555288950603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,65536,0.140592892964681
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,8192,0.014537778165605334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,7168,0.013198222551080914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,12288,0.13614400227864584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,16384,0.1847804387410482
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,10240,0.11583377255333795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,6144,0.012176888684431711
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,8192,0.09460888968573676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,5120,0.01185155577129788
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,4096,0.010083555347389644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,3584,0.009648888475365108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,7168,0.08512533373302883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,6144,0.07496355639563666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,5120,0.06467555628882514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,3072,0.008845333423879411
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,4096,0.05468533436457316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,3584,0.0495324432849884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,2048,0.006993778049945831
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,3072,0.044928001032935254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,65536,1.767024040222168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,2560,0.007983999947706858
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,1536,0.00923466682434082
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,768,0.005660444498062134
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,1024,0.006773333168692059
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,2048,0.034394668208228216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,2560,0.03974399964014689
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,1536,0.029861334297392104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,1024,0.02437422176202138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,512,0.004446222136418025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,256,0.0032542221662071017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,128,0.00258933338854048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,64,0.0021395554973019496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,768,0.02204977803760105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,512,0.01902222302224901
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,256,0.016678222351604037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,32,0.002327999927931362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,128,0.015032000011867948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,64,0.014276444911956787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,16384,0.01646222174167633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,65536,0.05531822310553657
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,32,0.014346667461925082
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,12288,0.013613333304723104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,10240,0.012049777640236748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,8192,0.010449777874681683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,7168,0.009767110976907942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,16384,0.1820666657553779
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,10240,0.11531999376085068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,12288,0.13623555501302084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,6144,0.009000889129108852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,8192,0.09444622198740642
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,5120,0.008016000191370646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,7168,0.08471733331680298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,4096,0.00703288863102595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,3584,0.006768888897365994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,6144,0.07483377721574572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,5120,0.0645315580897861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,4096,0.05463288889990913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,3072,0.006308444258239534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,2560,0.00600888869828648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,3584,0.04943822158707512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,2048,0.005497777627574072
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,1536,0.008887111312813228
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,3072,0.044791999790403575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,2560,0.039641777674357094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,65536,1.7664470672607422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,1024,0.006500444478458828
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,768,0.005393777870469623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,2048,0.03423466616206699
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,1536,0.02946399980121189
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,512,0.0041742221348815495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,256,0.0029813332690132987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,1024,0.024497777223587036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,128,0.0023635555472638872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,64,0.002004444392191039
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,768,0.021889777647124395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,512,0.019139554765489366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,256,0.016326222154829238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,32,0.0021884443445338146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,128,0.014896889527638754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,64,0.014184888866212634
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,16384,0.01737866633468204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,65536,0.05864444706175062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,32,0.014173333843549093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,12288,0.014458666245142618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,10240,0.013071999781661563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,8192,0.007357333269384172
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,7168,0.0070168889231152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,12288,0.13596178425682917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,8192,0.09431733025444879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,10240,0.11569155587090386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,16384,0.18378755781385633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,6144,0.009200000100665623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,5120,0.008539555801285638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,4096,0.007359111474619971
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,7168,0.08479644192589654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,3584,0.0070257774657673305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,6144,0.07437155644098918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,3072,0.006993778049945831
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,5120,0.0647031135029263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,4096,0.05456088980038961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,2560,0.0063591111037466265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,2048,0.005727111051479976
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,3584,0.0495395561059316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,3072,0.044340444935692676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,1536,0.00518844442235099
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,1024,0.005746666755941179
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,2560,0.03940177626079983
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,2048,0.03420089019669427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,768,0.004830222162935468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,65536,1.7720862494574652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,1536,0.029530667596393164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,512,0.004022222426202563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,1024,0.024392000503010217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,128,0.0022542222092549005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,256,0.0029680000411139596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,768,0.021914665897687275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,64,0.0023484445280498928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,512,0.018927110566033255
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,32,0.0021395554973019496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,256,0.016456888781653512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,128,0.015077332655588785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,65536,0.030735111898846094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,16384,0.010889777706729041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,64,0.014344889256689282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,32,0.014010666145218743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,12288,0.0092613332801395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,10240,0.007141333487298753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,8192,0.006510222123728857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,7168,0.007610666255156199
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,12288,0.13623289267222086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,10240,0.11512000030941433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,8192,0.09472799963421291
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,16384,0.18140710724724662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,6144,0.006940444310506185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,5120,0.006406222366624408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,4096,0.005689777847793367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,7168,0.0843120018641154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,3584,0.0058666666348775225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,6144,0.07429155376222399
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,3072,0.005372444374693765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,5120,0.06469777557584974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,4096,0.054530666934119336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,2560,0.005512889060709212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,2048,0.0046728890803125174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,3584,0.04920533299446106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,1536,0.004256000121434529
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,3072,0.04433066646258036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,2560,0.039388444688585066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,1024,0.005597333527273602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,2048,0.034373333056767784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,65536,1.7737599478827581
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,768,0.00464088883664873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,1536,0.029151999288135107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,512,0.0036657779581016968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,256,0.0027840000887711844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,128,0.0022960000981887183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,1024,0.02407111061943902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,768,0.021846221552954778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,64,0.0021768889079491296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,256,0.016121778223249648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,32,0.0021457777669032416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,128,0.015015999476114908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,512,0.018780445059140522
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,64,0.014143110977278816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,32,0.014204444156752693
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,65536,1.7737671534220378
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,10240,1.3764675988091364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,12288,1.741574181450738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,8192,1.077108489142524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,16384,2.41011471218533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,10240,2.5215466817220054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,8192,2.0743865966796875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,7168,0.9780622058444552
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,12288,3.258691575792101
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,6144,0.8172071244981555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,7168,1.7786471048990886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,6144,1.5980461968315973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,16384,3.9500380622016062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,5120,1.31913333468967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,4096,0.5776409043206109
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,5120,0.7377297613355848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,4096,1.0310648812188041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,3584,0.8795590930514865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,3584,0.5484559800889757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,3072,0.47158580356174046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,2560,0.4090542263454861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,2048,0.35562933815850156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,1536,0.4233626789516873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,2048,0.5513920254177517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,1536,0.2954062090979682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,1024,0.3024933338165283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,3072,0.8065964380900065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,2560,0.6840320163302951
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,256,0.11329955524868435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,768,0.20858577887217203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,512,0.2001964383655124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,1024,0.23627021577623156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,128,0.05409155620468987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,64,0.029858665333853826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,32,0.025384889708624944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,256,0.16007733345031738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,768,0.2675715552435981
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,64,0.13828178246816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,512,0.1831973393758138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,128,0.1538862254884508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,32,0.14126133918762207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,12288,0.5033653577168783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,16384,0.6877786848280164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,10240,0.6163102255927192
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,12288,0.8649404313829211
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,16384,1.0060062408447266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,10240,0.3998640113406711
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,8192,0.4732773568895128
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,8192,0.3348284562428792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,7168,0.28592801094055176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,6144,0.26014934645758736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,7168,0.5167422294616699
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,6144,0.43305688434176975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,5120,0.36031466060214573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,4096,0.2960088782840305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,5120,0.21475822395748564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,3584,0.2544524404737684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,4096,0.1679991086324056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,3072,0.22021067142486572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,3584,0.1649057732688056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,3072,0.14047822687360975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,2048,0.09913155767652725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,2560,0.11447289254930283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,2048,0.1458453337351481
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,2560,0.18918044037289092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,1536,0.08591110838784112
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,768,0.05370222197638618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,512,0.03425866696569655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,1536,0.11103200250201756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,256,0.01718311177359687
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,768,0.06363111072116427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,128,0.01237511138121287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,1024,0.07422311438454522
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,64,0.010616888602574667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,1024,0.0716808901892768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,512,0.05782310830222236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,32,0.008997333546479544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,64,0.04336000151104397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,256,0.05054488778114319
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,65536,2.9067876603868275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,32,0.04327999883227878
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,128,0.04597511225276523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,12288,0.4775102403428819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,65536,3.760136922200521
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,10240,0.5359830856323242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,16384,0.8771048651801215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,16384,0.6950097613864474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,12288,0.7616577678256564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,8192,0.2974231243133545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,10240,0.38207822375827366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,7168,0.37246934572855633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,8192,0.4535644319322374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,5120,0.2784915500217014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,6144,0.3502177662319607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,7168,0.27292532391018337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,4096,0.2332622210184733
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,4096,0.1473031176461114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,5120,0.1938533385594686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,6144,0.2467493216196696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,3072,0.1701671149995592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,3584,0.20428533024258086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,3584,0.13331645064883763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,3072,0.11954132715861003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,2048,0.11301422119140625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,2560,0.1417671177122328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,1536,0.08219999737209745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,2560,0.11165600352817112
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,768,0.046684443950653076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,1536,0.0779928896162245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,1024,0.060143113136291504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,2048,0.09392711189058091
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,1024,0.061428447564442955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,256,0.01668711172209846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,128,0.011914666328165265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,768,0.05653422408633762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,512,0.027811557054519653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,512,0.04945866598023308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,64,0.009719111025333405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,256,0.04299555553330315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,32,0.0075599998235702515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,128,0.039084444443384804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,32,0.036898665957980685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,64,0.038726223839653864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,65536,2.8044728173149953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,65536,3.050530751546224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,16384,0.4693031311035156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,16384,0.6399200227525499
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,12288,0.3485902150472005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,12288,0.5312026871575249
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,10240,0.40489154391818577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,8192,0.31679288546244305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,10240,0.28093600273132324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,8192,0.22411643134223092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,7168,0.27870223257276744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,7168,0.19378488593631318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,6144,0.17567111386193168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,4096,0.14871022436353895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,5120,0.2127751111984253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,6144,0.2560817665523953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,5120,0.13777511649661595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,4096,0.10775199863645767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,3584,0.10174666510687934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,3072,0.10663111342324151
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,2560,0.0905146665043301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,3072,0.08814933564927842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,2560,0.07907378011279635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,3584,0.13302221563127306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,2048,0.07204444540871514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,1536,0.05455999904208713
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,2048,0.06925777594248454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,1024,0.04164088765780131
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,1024,0.049157334698571094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,512,0.01854222185081906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,1536,0.06014044417275322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,768,0.025504888759719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,65536,2.194589402940538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,256,0.012701333396964602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,128,0.0087013335691558
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,256,0.03612622287538316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,65536,2.4942283630371094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,64,0.030932443009482488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,32,0.005680000202523337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,64,0.00702311098575592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,512,0.04049866729312473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,128,0.03278311093648275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,768,0.045496000183953174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,32,0.032087110810809664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,12288,0.35836089981926816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,12288,0.4142017894321018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,16384,0.5329057905409071
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,16384,0.46106576919555664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,10240,0.2935182253519694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,10240,0.32164356443617076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,8192,0.23709689246283638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,6144,0.19766222106085884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,7168,0.256494230694241
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,8192,0.2528897656334771
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,5120,0.14845866627163357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,7168,0.2173475556903415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,6144,0.21213155322604707
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,3584,0.10078133477105035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,4096,0.12969688574473062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,4096,0.1397386656867133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,3584,0.12697599993811712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,5120,0.16057688660091823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,3072,0.101310220029619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,2560,0.08709422085020278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,2048,0.07093422280417548
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,1536,0.05514311128192478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,3072,0.09528088569641113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,2048,0.07475555605358548
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,2560,0.09316533141665989
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,1024,0.03194577826393975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,1536,0.06283199787139893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,768,0.027614222632514104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,256,0.011983999775515662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,512,0.019493333167499967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,128,0.00816000004609426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,768,0.04672355453173319
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,1024,0.05432888865470886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,64,0.0063031112982167145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,32,0.00544799988468488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,65536,1.9387484656439886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,256,0.03915199968549941
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,128,0.03464711043569777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,65536,1.9622781541612413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,512,0.04457777738571167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,64,0.032808889945348106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,32,0.031473779016070895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,10240,0.25409065352545845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,12288,0.32726044125027126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,10240,0.2762906551361084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,12288,0.3341840108235677
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,16384,0.45769156350029844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,16384,0.4326542218526204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,8192,0.21046133836110434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,7168,0.18702756034003365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,8192,0.22298667165968153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,5120,0.1443635490205553
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,6144,0.1816542281044854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,6144,0.16652444998423258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,7168,0.18917244010501436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,5120,0.1306497785780165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,3584,0.09848266839981079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,3072,0.10665155781639947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,3584,0.10598222414652507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,3072,0.08486488792631362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,2560,0.08874755435519749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,2560,0.07349599732293023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,4096,0.12177688545650905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,2048,0.06247199906243218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,2048,0.07576533158620198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,1024,0.031042665243148804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,1536,0.054486221737331815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,768,0.02436888880199856
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,1536,0.05328355564011467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,1024,0.04506311151716444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,512,0.019285332825448778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,256,0.011800888511869641
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,4096,0.11873866452111138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,768,0.041004445817735456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,65536,1.7760595745510523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,128,0.008052444292439355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,512,0.036363555325402155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,64,0.006226666685607698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,32,0.005394666559166378
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,256,0.032431999842325844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,64,0.02903111113442315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,32,0.028667555914984807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,128,0.030232889784706965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,65536,1.9343581729465062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,16384,0.37955466906229657
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,12288,0.29000266393025714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,16384,0.40952266587151426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,12288,0.31899645593431264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,10240,0.23281778229607475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,8192,0.18798489040798613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,10240,0.24920354949103463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,8192,0.19034932719336617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,7168,0.1615813308291965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,7168,0.19793332947625053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,6144,0.1588675578435262
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,5120,0.1419137716293335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,4096,0.11232444975111221
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,4096,0.09983288579516941
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,3584,0.09364977810117935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,6144,0.14806666639116076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,5120,0.11562844117482503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,3072,0.08483288685480754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,2560,0.08201333549287584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,3584,0.09263466464148627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,3072,0.0744382209248013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,65536,1.4873991012573242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,1536,0.05634044276343452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,2048,0.05877955754597982
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,2560,0.06701244248284234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,1536,0.0498888889948527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,2048,0.07374755541483562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,768,0.02681866619322035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,512,0.019097778532240126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,256,0.011801777614487542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,128,0.008067555725574493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,1024,0.042916443612840444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,1024,0.03486844566133287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,512,0.03467377689149644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,256,0.0295688874191708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,768,0.038154668278164335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,32,0.00461422238085005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,64,0.006022222340106964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,128,0.0284862220287323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,64,0.02715733316209581
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,32,0.026389333936903212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,65536,1.8898311191134984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,16384,0.2969697846306695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,12288,0.27015378740098744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,16384,0.3818648921118842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,10240,0.199800001250373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,8192,0.15284355481465658
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,10240,0.22044889132181802
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,12288,0.29040800200568306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,6144,0.10563021898269653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,7168,0.11958132849799262
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,8192,0.16670400566524932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,7168,0.14378755622439915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,5120,0.09090310997433132
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,4096,0.07369510995017158
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,5120,0.11298399501376682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,6144,0.13275999493069118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,3072,0.053968889845742114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,3584,0.0814195540216234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,3584,0.06279377804862128
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,4096,0.08963288863499959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,2560,0.05046400096681383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,2048,0.03743022349145677
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,3072,0.07572978072696261
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,1536,0.02546666728125678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,65536,1.2902835210164387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,2048,0.056926223966810435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,768,0.015392000476519266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,1536,0.04892266790072123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,512,0.011376889215575324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,1024,0.04163466559516059
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,256,0.007440889047251807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,768,0.03897777862019009
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,2560,0.06449333164427015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,128,0.005482666608360078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,64,0.00452977791428566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,512,0.033194667763180204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,32,0.003960888832807541
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,256,0.029255112012227375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,64,0.025516443782382544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,1024,0.01843377782238854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,32,0.025760889053344727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,128,0.027263111538357202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,65536,1.834664026896159
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,16384,0.24395200941297743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,12288,0.1956853336758084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,16384,0.25176800621880424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,8192,0.12587911552853054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,12288,0.18432088692982992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,10240,0.124981337123447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,10240,0.15092533164554173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,8192,0.09706844223870172
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,6144,0.08635999759038289
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,7168,0.09917689032024807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,6144,0.07551911142137316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,7168,0.0875564416249593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,4096,0.059551113181644015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,5120,0.073125335905287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,5120,0.06590044498443604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,3584,0.05240177777078417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,3072,0.04491200049718221
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,65536,1.0248408847384982
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,2560,0.03975555631849501
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,2048,0.033168001307381526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,3072,0.04677599999639723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,3584,0.05125511023733351
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,4096,0.0559084415435791
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,1536,0.02524799936347538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,2560,0.04179999894566006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,768,0.015305777390797934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,2048,0.037552889850404524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,1024,0.01819644371668498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,1536,0.03315555718210008
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,512,0.011143999795118967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,1024,0.02804622385236952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,256,0.007282666862010956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,768,0.025458667013380263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,64,0.004366222355100843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,512,0.023165333602163527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,32,0.003710222327046924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,128,0.005351999981535806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,65536,1.4710328843858507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,256,0.020541333489947848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,128,0.01867999964290195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,64,0.017961778574519686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,32,0.018357333209779527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,12288,0.17299555407630074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,10240,0.13519110944535997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,16384,0.21806932820214164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,12288,0.15629155106014675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,16384,0.23066666391160753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,10240,0.1173057821061876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,8192,0.11190133624606663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,7168,0.08957244290245904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,8192,0.09358488851123387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,6144,0.07871911260816786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,5120,0.06832622157202826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,7168,0.08294488986333211
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,6144,0.0749466684129503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,4096,0.05791022380193075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,3584,0.05101866523424784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,3072,0.044722666343053184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,4096,0.05562577644983927
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,3584,0.0506257779068417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,65536,0.8934080335828992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,2048,0.03266577919324239
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,3072,0.04596977763705783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,5120,0.06528533167309232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,2560,0.04144355654716492
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,2560,0.037816001309288874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,1024,0.0181377778450648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,768,0.014711999230914645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,512,0.010783111055692038
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,2048,0.03686044282383389
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,256,0.007110222346252865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,1024,0.02760533326201969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,1536,0.027863999207814533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,768,0.025063999825053748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,128,0.0052391112678580815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,1536,0.032411555449167885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,512,0.022620444496472675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,32,0.0033297776761982176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,256,0.020015999674797058
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,64,0.004467555632193883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,128,0.018414222531848483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,64,0.01757066614098019
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,32,0.018023111753993563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,16384,0.19308000140719941
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,12288,0.15032000011867946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,10240,0.11947022544013129
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,65536,1.4258533053927953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,12288,0.1386622190475464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,16384,0.21916445096333823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,8192,0.1021582219335768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,7168,0.090393775039249
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,10240,0.11214755641089545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,6144,0.08314133352703519
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,7168,0.082168890370263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,5120,0.07465600305133395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,8192,0.09273333019680446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,5120,0.06400889158248901
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,6144,0.07310933536953397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,65536,0.7917635705735949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,3584,0.05055377880732218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,4096,0.05801244576772054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,4096,0.055042667521370776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,2048,0.032604446013768516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,2560,0.03761955433421665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,3584,0.05032444331381056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,2560,0.040766222609413996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,1536,0.025983111725913152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,1024,0.0181377778450648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,3072,0.04659733176231384
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,2048,0.0365573333369361
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,1536,0.03196533189879523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,768,0.014652444256676568
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,512,0.010683555569913654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,1024,0.02754666739039951
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,256,0.007072889142566257
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,3072,0.046338667472203575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,128,0.00519822206762102
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,512,0.02203111184967889
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,768,0.02511022157139248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,256,0.019640889432695176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,64,0.004309333446953031
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,32,0.0031155554784668815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,128,0.017608889275126986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,64,0.01757244434621599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,65536,1.4010284211900499
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,32,0.01749955614407857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,12288,0.12903643978966609
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,16384,0.16047999593946669
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,10240,0.10231733322143555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,16384,0.2002320024702284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,8192,0.08416710959540473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,12288,0.13305421670277914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,8192,0.0901413361231486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,7168,0.06616177823808458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,6144,0.05685422155592176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,7168,0.08079822195900811
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,5120,0.04983555608325534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,10240,0.11031822363535564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,65536,0.6643990940517849
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,6144,0.07163911395602755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,4096,0.0410444438457489
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,4096,0.05288444293869866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,3072,0.029654221402274236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,3584,0.0359173317750295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,5120,0.062005334430270724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,3584,0.05051022105746799
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,2048,0.021137777302000258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,1536,0.0177528891298506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,2560,0.026239110363854304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,2560,0.039289779133266874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,2048,0.03445955448680454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,3072,0.04381155636575487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,768,0.010073777702119615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,1024,0.012479110724396177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,1536,0.030388444662094116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,512,0.0074577778577804565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,1024,0.025213332639800176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,128,0.004138666722509596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,256,0.005447111195988125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,768,0.022873777482244704
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,512,0.0203164451652103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,64,0.0035413333939181436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,256,0.018208000395033095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,128,0.016226665841208566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,32,0.0029279999434947968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,64,0.015634665886561077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,32,0.01585422290696038
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,12288,0.08984533283445571
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,16384,0.11690133147769505
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,10240,0.07446666558583577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,65536,1.3805191251966689
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,8192,0.06699022319581774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,16384,0.18817955917782256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,12288,0.1292133331298828
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,7168,0.055824889077080615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,10240,0.10939111312230428
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,6144,0.049118220806121826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,7168,0.07961066563924153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,8192,0.08910933468076919
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,6144,0.070942223072052
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,65536,0.5559555689493815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,4096,0.0345164438088735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,5120,0.06135111384921604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,3072,0.026399999856948853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,5120,0.04531911015510559
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,2560,0.021781333618693884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,4096,0.05207822057935926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,3584,0.04780266682306925
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,2048,0.017789334058761597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,3584,0.03048444456524319
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,3072,0.04280266828007168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,2560,0.038438220818837486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,1536,0.014200000299347771
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,1024,0.010433778166770935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,2048,0.03369599911901686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,512,0.006808888994985157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,1536,0.029363556040657893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,1024,0.024461333950360615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,256,0.005050666630268097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,768,0.022291556000709534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,768,0.008739555875460306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,128,0.0038906666967603895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,64,0.0033964444365766314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,512,0.019523556033770244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,32,0.003080000066094928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,256,0.01720977822939555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,128,0.015574221809705099
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,32,0.015213333898120456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,64,0.014800000521871777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,16384,0.08686221970452203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,12288,0.06924533181720309
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,10240,0.05845955345365736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,12288,0.12799021932813856
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,65536,1.3699368370903864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,16384,0.17451022730933297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,10240,0.10759822527567546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,8192,0.04673688941531711
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,65536,0.43807554244995117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,7168,0.042040887806150645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,6144,0.03719022207789951
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,5120,0.030312889152103003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,4096,0.025137777129809063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,5120,0.0606844425201416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,3584,0.02286400066481696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,7168,0.07888799905776978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,4096,0.054125332170062594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,8192,0.08878311183717515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,3072,0.02038755516211192
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,6144,0.0698853333791097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,2048,0.01672266589270698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,2560,0.019296000401178997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,3584,0.04708977871470981
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,1536,0.013443555268976422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,2560,0.03778310947948032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,1024,0.010075555907355415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,2048,0.03319822086228265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,3072,0.0421013335386912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,768,0.008488888541857401
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,512,0.006592000110281839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,1536,0.029135111305448744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,1024,0.024131556351979572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,256,0.004888888862397936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,768,0.021594666772418555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,128,0.0038559999730851916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,64,0.0030213333666324615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,512,0.018686221705542672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,32,0.0028373334142896864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,128,0.015150222513410779
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,256,0.016470222009552848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,64,0.014273777604103088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,32,0.014257777068350049
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,12288,0.04658311274316576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,16384,0.10795999897850884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,10240,0.03975911272896661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,65536,1.3743893305460613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,8192,0.031818668047587075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,16384,0.16702577802870008
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,12288,0.12600266933441162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,65536,0.30131644672817653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,7168,0.027091556125217017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,10240,0.10676178005006577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,6144,0.024170666933059692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,5120,0.020871111088328894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,4096,0.018254222141371835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,7168,0.0787022246254815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,3584,0.01722044414944119
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,5120,0.06039555867513021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,4096,0.051254222790400185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,3072,0.017299556069903903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,8192,0.0880311131477356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,2560,0.015590222345458137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,6144,0.06970577769809298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,3584,0.046635554896460645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,2048,0.013643555343151093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,3072,0.042018665207756885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,1536,0.010168000227875179
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,1024,0.007737778127193451
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,768,0.006533333410819371
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,2048,0.032920890384250216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,2560,0.037589334779315524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,1536,0.02830844455295139
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,512,0.005414222263627582
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,1024,0.023560888237423364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,128,0.0033128888656695685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,256,0.0041351111398802865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,768,0.021136000752449036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,512,0.0184879998366038
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,64,0.0026791110220882627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,256,0.016162667009565566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,32,0.0024622221373849446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,128,0.014873777826627096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,64,0.014170666535695394
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,32,0.01426577733622657
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,12288,0.03599377804332309
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,16384,0.045124444696638316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,65536,0.24858578046162924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,10240,0.02629599968592326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,8192,0.022952889402707417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,12288,0.12583288881513807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,16384,0.16629421710968018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,65536,1.3852977752685547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,7168,0.021150221427281696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,10240,0.10724800162845188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,5120,0.01741066740618812
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,8192,0.08806311421924168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,6144,0.01904000010755327
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,4096,0.01800622211562263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,3584,0.016514667206340365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,7168,0.0790426664882236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,5120,0.06081510914696587
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,6144,0.07003644439909193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,3072,0.015000889698664347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,4096,0.05128799875577291
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,3584,0.04670666654904684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,2048,0.01072088877360026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,2560,0.013533333109484779
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,1536,0.009027555584907532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,3072,0.04204710986879137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,1024,0.006785777707894643
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,2560,0.03765866822666592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,768,0.00590133335855272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,2048,0.032864888509114586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,512,0.004867555366622077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,1536,0.02843733297453986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,256,0.003903999924659729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,1024,0.023560888237423364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,768,0.02128355536195967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,128,0.003105777833196852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,512,0.01854044364558326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,64,0.002534222271707323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,32,0.002346666736735238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,256,0.016220443778567843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,128,0.014892444014549255
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,16384,0.03199377655982971
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,64,0.014215999179416232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,32,0.01423111061255137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,10240,0.019655111763212416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,12288,0.0222080002228419
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,65536,0.19405955738491484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,8192,0.01661688917213016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,7168,0.016334222422705758
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,16384,0.16438489490085179
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,12288,0.12556889322068956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,6144,0.016280000408490498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,10240,0.1069635550181071
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,8192,0.08837422397401597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,5120,0.014004444082578024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,65536,1.4008070627848308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,4096,0.011928888658682505
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,7168,0.07935377624299791
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,3584,0.01161688897344801
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,6144,0.06999644305970934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,3072,0.01051644484202067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,5120,0.060752001073625356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,4096,0.051516443490982056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,3584,0.04692711101637947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,2560,0.011716444459226398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,2048,0.009826666778988307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,1536,0.008167111211352879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,1024,0.0063440000845326324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,3072,0.04199644592073229
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,2560,0.03766311208407084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,768,0.005586666779385672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,512,0.004584889031118817
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,2048,0.03305955396758185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,1536,0.028544889556037054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,1024,0.023731556203630235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,256,0.003719999972316954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,128,0.002962666667169995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,64,0.002324444345302052
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,768,0.021224000387721594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,512,0.018771555688646104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,32,0.0026657777941889236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,256,0.016348444753222995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,128,0.015101333459218344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,64,0.014455111490355598
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,16384,0.019389332996474374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,65536,0.07288266552819146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,32,0.014127110441525778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,12288,0.01621955633163452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,10240,0.014408888088332282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,8192,0.012526222401195101
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,7168,0.011304000185595619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,16384,0.16322133276197645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,10240,0.10654577944013809
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,6144,0.011543110840850405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,12288,0.12541600068410239
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,5120,0.010090666512648264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,4096,0.008778666456540426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,8192,0.0884808897972107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,65536,1.4025768703884547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,6144,0.0700062248441908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,7168,0.07899910873836942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,5120,0.060721774895985924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,3584,0.008440000315507254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,2560,0.007139555282062954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,4096,0.05139644609557258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,2048,0.0064479998416370815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,3072,0.007800000409285228
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,3072,0.04207466708289253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,3584,0.046945777204301625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,2560,0.03772888912094964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,1536,0.00907466643386417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,1024,0.006630222416586346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,768,0.005519111123349931
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,512,0.004280000097221798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,2048,0.03288355469703674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,1536,0.02850488821665446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,256,0.0031155554784668815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,1024,0.023749333288934495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,768,0.021386666430367365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,128,0.0024968888610601425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,512,0.018741332822375827
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,64,0.0020711111525694528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,32,0.0022479999396536085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,256,0.016120889120631747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,128,0.015020444989204407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,16384,0.013772444592581855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,65536,0.0432168874475691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,64,0.014427555931939019
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,32,0.014300444059901767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,12288,0.011546666423479715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,10240,0.010291555689440833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,8192,0.009161777794361115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,7168,0.00832444429397583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,16384,0.16283022032843694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,12288,0.12520266903771296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,10240,0.10685511430104573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,6144,0.0076453329788313965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,8192,0.08838755554623073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,5120,0.007058666812049017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,65536,1.4023449156019423
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,4096,0.006383111079533895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,7168,0.0788408915201823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,3072,0.00600622221827507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,3584,0.006256000035338932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,5120,0.06054311328464084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,6144,0.0698302189509074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,4096,0.051538666089375816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,2560,0.005607999861240387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,3584,0.04689866635534498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,2048,0.005041777673694822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,1536,0.005052444421582752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,1024,0.006450666735569636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,3072,0.04181333382924398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,2048,0.033024887243906655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,2560,0.03750577900144789
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,768,0.0052693333062860705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,1536,0.028416888581381902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,512,0.004121777911980947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,256,0.0029582221888833572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,1024,0.023404444257418316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,128,0.0023422222584486008
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,768,0.020970667401949566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,512,0.018585777944988675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,64,0.0019822222077184254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,32,0.002177777803606457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,256,0.0162791113058726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,128,0.014992000328169929
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,65536,0.04392710990375943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,16384,0.014465777410401238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,64,0.014330666926172046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,32,0.014183110660976835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,12288,0.012216889195972018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,10240,0.010984000232484607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,8192,0.0092622223827574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,7168,0.00906577789121204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,16384,0.1627662181854248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,8192,0.08829955259958903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,10240,0.10654666688707139
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,12288,0.12522399425506592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,5120,0.007705777883529663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,7168,0.07872711287604438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,65536,1.406736903720432
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,4096,0.006787555499209299
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,6144,0.00812355594502555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,6144,0.06980088684293959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,5120,0.060544888178507485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,3584,0.0069288888739215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,3072,0.006416888700591193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,2560,0.0058577776783042485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,4096,0.05115555392371284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,2048,0.005567111074924469
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,3584,0.04659644431538052
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,1536,0.004909333255555895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,1024,0.00563822231358952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,3072,0.04179911149872673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,768,0.0047253333032131195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,2560,0.03750844465361701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,2048,0.032841778463787503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,512,0.0037582222786214617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,1536,0.028036445379257202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,256,0.0028133332315418455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,1024,0.023436443673239812
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,128,0.0022533333135975734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,768,0.020896888441509668
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,64,0.0021848889688650766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,512,0.018560000591807894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,256,0.016206221448050607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,32,0.002151111140847206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,128,0.01494311127397749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,65536,0.024273778001467388
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,64,0.01401600076092614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,16384,0.008079110748238033
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,32,0.014127110441525778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,12288,0.007309333317809635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,10240,0.00675999994079272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,8192,0.00628355559375551
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,7168,0.007075555622577667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,12288,0.12493688530392116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,10240,0.10683644480175442
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,16384,0.1627875566482544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,8192,0.087918221950531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,6144,0.006426666759782367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,5120,0.005954666684071223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,4096,0.005769777629110549
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,7168,0.07892088757620917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,3584,0.005511111269394557
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,6144,0.06987910800509982
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,5120,0.06015288829803467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,65536,1.4079590903388128
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,3072,0.005175999883148405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,2560,0.004693333473470476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,4096,0.05129422081841362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,2048,0.004605333424276776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,1536,0.004061333421203825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,3584,0.046704888343811035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,3072,0.04200889004601372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,1024,0.005535111245181825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,768,0.004581333448489507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,2560,0.037208000818888344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,2048,0.03262133399645487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,1536,0.027999109692043726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,512,0.0036311112344264984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,256,0.0027315556589100095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,1024,0.02353688908947839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,128,0.002242666565709644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,64,0.0021564445147911706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,768,0.02106933295726776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,512,0.01825777855184343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,256,0.01591999994383918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,32,0.0021564445147911706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,128,0.014742222097184924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,64,0.014215999179416232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,32,0.014112888111008538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,65536,1.405326207478841
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,10240,1.4551849365234375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,12288,1.7401840421888564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,16384,2.2409129672580295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,10240,2.242450714111328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,8192,1.1289982265896268
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,8192,1.8428240882025824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,7168,0.9547804726494683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,12288,2.9247430165608725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,5120,0.658931573232015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,6144,0.7907262378268771
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,16384,3.5332639482286243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,7168,1.6568373574150934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,5120,1.1611608929104276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,4096,0.5719208717346191
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,4096,0.9477590984768338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,6144,1.4681101904975042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,3584,0.5093022452460395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,3584,0.8188648753696017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,3072,0.4232906765407986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,3072,0.7104844517178006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,2048,0.30733243624369305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,2560,0.3882417678833008
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,1024,0.2726444403330485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,2560,0.6379929118686253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,1536,0.4010417726304796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,768,0.21610577901204428
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,1536,0.259680880440606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,256,0.0735093355178833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,2048,0.5169395340813531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,128,0.03976800044377645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,64,0.022145777940750122
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,512,0.13071821795569524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,768,0.16246399614546034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,256,0.11328177981906468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,32,0.017190222938855488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,1024,0.1960737837685479
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,64,0.09762577878104316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,512,0.15221688482496473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,32,0.10108800066841973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,128,0.10502044359842937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,10240,0.5483013259039985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,12288,0.5304764641655816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,16384,0.6910799874199761
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,12288,0.7471102078755697
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,16384,0.9459564420912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,10240,0.3839306566450331
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,7168,0.39224176936679417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,8192,0.4723075760735406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,8192,0.3211217721303304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,7168,0.272436433368259
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,6144,0.3477608892652724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,6144,0.22187377346886528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,5120,0.2861093415154351
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,5120,0.21114667256673178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,3584,0.2060879998736911
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,4096,0.23891109890407988
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,4096,0.15830755233764648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,2560,0.13953155941433376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,3072,0.1946035491095649
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,65536,2.2342327965630426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,3584,0.13912088341183132
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,2048,0.10788089036941528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,3072,0.12080444229973687
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,2560,0.10515199767218696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,1536,0.08145688639746772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,768,0.04345777630805969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,1536,0.07406222158008151
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,2048,0.0897208915816413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,1024,0.05859555800755819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,512,0.029835555288526747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,256,0.013516444298956128
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,1024,0.06242666641871134
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,512,0.04095822241571214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,64,0.0069466663731469055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,768,0.04690044456058078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,256,0.0369759996732076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,128,0.008728889127572378
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,32,0.005921777751710679
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,128,0.03358133302794562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,64,0.031472888257768415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,32,0.03144800000720554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,12288,0.36969865692986387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,10240,0.4457937876383464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,16384,0.49087201224433047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,16384,0.6919440163506402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,65536,3.7132593790690103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,12288,0.5843937661912706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,10240,0.30496178732977974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,8192,0.3543733225928412
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,7168,0.2113120026058621
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,7168,0.3109110991160075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,8192,0.24518932236565483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,6144,0.28303999370998806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,6144,0.1885626713434855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,5120,0.22395110130310059
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,4096,0.17913956112331816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,4096,0.1168737808863322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,3584,0.15475200282202825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,3072,0.1291333304511176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,5120,0.16599555810292563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,2560,0.10851733552085029
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,2048,0.08788000212775336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,2560,0.08395466539594863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,3584,0.1061484416325887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,1536,0.067958222495185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,2048,0.07558311356438531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,3072,0.1000479989581638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,65536,1.8651902940538194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,1024,0.04627289043532478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,1024,0.049031111929151744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,256,0.012175111307038201
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,512,0.01921866668595208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,768,0.041238221857282854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,768,0.034444444709353976
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,128,0.008229333493444655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,512,0.03711822297837999
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,256,0.03291555576854282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,1536,0.06360533502366808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,64,0.0064151109092765385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,32,0.005364444520738389
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,128,0.030271109607484605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,64,0.028510222832361858
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,32,0.02878311276435852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,16384,0.48340712653266055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,65536,2.6262327829996743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,12288,0.36143022113376194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,12288,0.45714934666951496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,16384,0.6212319797939724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,8192,0.2911270989312066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,7168,0.25131199094984263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,8192,0.23605423503451875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,10240,0.3902791076236301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,7168,0.2134631077448527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,10240,0.2971564398871528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,6144,0.2237128946516249
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,5120,0.18337689505683052
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,6144,0.17788533369700113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,4096,0.11516355143653022
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,5120,0.1530204481548733
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,4096,0.18208444118499756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,3072,0.1261448860168457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,3072,0.09289333555433486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,3584,0.10347821977403428
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,3584,0.1588008933597141
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,2560,0.11516355143653022
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,1536,0.07226577732298109
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,2048,0.08637866708967422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,65536,1.8460328843858507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,2048,0.06999733050664265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,1024,0.0503786669837104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,1536,0.06072533130645752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,2560,0.08595289124382867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,768,0.024676443801985845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,1024,0.043840001026789345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,512,0.019371555911170114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,256,0.011892444557613797
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,768,0.03945599993069967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,128,0.008060444560315874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,64,0.006212444355090459
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,512,0.03622755408287048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,65536,2.2494710286458335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,128,0.02938400043381585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,64,0.02693333393997616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,256,0.03197777933544583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,32,0.02737244466940562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,32,0.004877333425813251
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,12288,0.26308978928460014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,16384,0.35143287976582843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,16384,0.5010088814629449
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,12288,0.39790844917297363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,10240,0.28273510932922363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,8192,0.2545688947041829
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,10240,0.21062666840023467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,7168,0.14391644795735678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,8192,0.17417600419786242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,7168,0.20047999752892387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,5120,0.1339306698905097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,6144,0.11425511042277019
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,4096,0.10873244206110637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,3584,0.09339822000927395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,6144,0.19065244992574057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,5120,0.10149066978030735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,3072,0.08101244767506917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,2560,0.07613955603705512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,3072,0.06238310866885715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,4096,0.08337689108318752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,3584,0.07482666439480253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,65536,1.4872390958997939
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,2048,0.061962664127349854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,1536,0.04282844397756788
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,1024,0.02260622216595544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,1536,0.03460089034504361
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,768,0.015444444285498725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,2048,0.047762665483686656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,512,0.01201866649919086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,256,0.007782222496138678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,65536,1.7894053988986547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,1024,0.02960444490114848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,2560,0.055659552415212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,64,0.004709333181381226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,128,0.005596444424655702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,32,0.004292444636424382
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,768,0.027424888478385076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,256,0.02260622216595544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,64,0.019730667273203533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,32,0.02036711076895396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,128,0.021007110675175984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,512,0.02539288832081689
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,12288,0.3199191093444824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,10240,0.24623288048638237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,16384,0.31521156099107533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,12288,0.2580515543619792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,16384,0.4067093266381158
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,10240,0.19828089078267416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,8192,0.20219643910725912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,7168,0.1766035556793213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,8192,0.13770400153266057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,7168,0.11786399947272407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,6144,0.16339022583431667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,6144,0.10207555691401164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,5120,0.13051644961039224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,4096,0.09581155247158474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,3584,0.08340089188681708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,4096,0.0703279972076416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,3072,0.07239555650287204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,3584,0.06381422281265259
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,5120,0.08790222141477798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,2560,0.061117331186930336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,2048,0.04969244533114963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,3072,0.05693333016501533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,1536,0.04067466656366984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,65536,1.2414169311523438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,2560,0.051048888100518115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,1024,0.019719110594855416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,2048,0.039042668210135564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,768,0.015056888262430826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,512,0.011201777391963534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,256,0.007339555356237624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,1536,0.03278933299912347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,128,0.005453333258628845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,65536,1.5964213477240667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,1024,0.028806222809685603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,256,0.02147377696302202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,128,0.019856888386938307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,32,0.004006222304370669
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,64,0.004498666773239772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,768,0.026183111800087824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,32,0.018779555956522625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,512,0.02403822210099962
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,64,0.018719111879666645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,12288,0.27369244893391925
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,16384,0.3487679958343506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,12288,0.21564088927374947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,16384,0.29100177023145885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,10240,0.23430310355292427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,8192,0.17526843812730578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,8192,0.11484621630774604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,7168,0.15171910656823054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,6144,0.12454933590359157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,10240,0.15365778075324163
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,7168,0.10599999957614475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,6144,0.0878702203432719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,4096,0.08459644185172187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,3584,0.07214399841096666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,4096,0.06734577814737956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,3072,0.06323377953635322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,3584,0.05724800295299954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,65536,1.1378595564100478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,2560,0.053293334113227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,65536,1.3682390848795574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,3072,0.054336888922585376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,5120,0.08030933141708374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,5120,0.11298844549391006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,2048,0.04376177655325996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,2048,0.035976890060636736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,1536,0.03746755586730109
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,1024,0.018234666850831773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,512,0.010847110715177325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,1536,0.03146133489078946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,768,0.025072889195548162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,768,0.014785778191354541
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,512,0.022758222288555566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,1024,0.02755111124780443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,128,0.005365333209435145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,256,0.020121778051058452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,32,0.003724444243643019
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,64,0.004410666724046071
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,256,0.007169778148333232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,128,0.018575110369258456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,32,0.018068444397714403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,64,0.018200889229774475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,2560,0.045808888143963285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,12288,0.16274578041500515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,12288,0.2382240030500624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,16384,0.33479110399882
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,16384,0.23925778600904676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,10240,0.20009422302246094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,8192,0.15098844634162054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,10240,0.12121421760982937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,7168,0.12200889322492812
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,7168,0.08522933059268528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,8192,0.10631733470492893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,6144,0.10726844602160984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,5120,0.09524622228410508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,4096,0.07765689161088732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,3584,0.06544622447755602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,6144,0.07626399728986952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,65536,1.0218435923258464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,5120,0.06615910927454631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,4096,0.05604888995488485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,65536,1.1325741873847115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,3072,0.06308799982070923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,2560,0.061179558436075844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,3584,0.054251554939481944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,2048,0.03894933395915561
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,1536,0.029472000069088403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,1024,0.017942221628295053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,768,0.014631110760900708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,2048,0.03520888752407498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,2560,0.03969688879119025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,512,0.010677333507272931
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,1536,0.030847999784681533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,256,0.007048889166778988
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,3072,0.04385333259900411
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,512,0.0217884447839525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,64,0.004326222257481681
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,128,0.0052266667286554975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,32,0.0033039999090962936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,768,0.02424000038041009
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,1024,0.026537777649031744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,128,0.01796800063716041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,64,0.017830222845077515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,32,0.01753422286775377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,256,0.019709333777427673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,10240,0.14992622534434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,12288,0.1919715536965264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,12288,0.11664177311791314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,10240,0.09506133529875015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,16384,0.2724115583631727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,8192,0.11266488499111599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,8192,0.07764444748560588
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,16384,0.18524621592627633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,7168,0.09745955467224121
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,6144,0.09415200021531846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,5120,0.07187910874684651
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,7168,0.0691244469748603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,6144,0.06134133206473457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,5120,0.05416799916161431
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,4096,0.059316443072425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,3584,0.053123556905322604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,3072,0.04636711213323805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,2560,0.03460444344414605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,3584,0.0369759996732076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,65536,0.926002714369032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,2048,0.02194044490655263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,4096,0.04456444581349691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,65536,0.8935502370198568
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,3072,0.03370755579736497
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,1536,0.01754666699303521
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,1024,0.012506666282812754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,2048,0.027116444375779893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,768,0.010206222534179688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,512,0.007981333467695449
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,2560,0.03049244483311971
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,256,0.0053093334039052325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,1024,0.020690666304694284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,128,0.00416711096962293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,1536,0.024121777878867254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,768,0.01919822229279412
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,512,0.017416000366210938
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,32,0.003003555453485913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,256,0.015828443898095023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,128,0.014270222849316068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,64,0.013930666777822705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,64,0.003638222399685118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,16384,0.1634720034069485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,12288,0.15801600615183511
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,32,0.014150222142537435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,16384,0.21391822232140434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,12288,0.10422933101654053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,10240,0.1305911143620809
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,10240,0.08686577611499363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,8192,0.07201066944334242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,8192,0.1197928852505154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,7168,0.09830933147006565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,7168,0.06368800004323323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,6144,0.0882728894551595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,65536,0.8128817876180013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,5120,0.07075377967622545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,4096,0.05346488952636719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,6144,0.05681955814361572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,3584,0.04706933432155185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,5120,0.04998666710323758
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,3072,0.03592888845337762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,2560,0.030351112286249798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,2048,0.019655111763212416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,4096,0.036660446061028376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,65536,0.8349519835578071
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,1536,0.01607644392384423
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,2560,0.029281778468026057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,1024,0.01149777736928728
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,1536,0.02280355493227641
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,768,0.009697777529557547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,3584,0.03492444422509935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,512,0.007234666910436418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,3072,0.03195555673705207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,1024,0.01960711181163788
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,256,0.005231110999981563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,128,0.004173333446184794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,2048,0.025789333714379206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,64,0.00350222239891688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,512,0.01685066686736213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,768,0.018263111511866253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,128,0.013829333086808523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,32,0.0033253334048721525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,64,0.013668444421556262
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,32,0.013576889203654395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,16384,0.19723733266194662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,256,0.014954666296641031
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,16384,0.14792354901631674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,12288,0.13193599383036295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,12288,0.09421510828865899
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,10240,0.10502666897243923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,8192,0.08747111426459418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,7168,0.0795653329955207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,8192,0.06652000215318468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,10240,0.08085688617494372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,7168,0.05871022409862942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,6144,0.07492711146672566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,5120,0.05782933367623223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,65536,0.7149973445468478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,6144,0.05251555641492208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,4096,0.046725332736968994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,3072,0.03134311238924662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,3584,0.04487199915779961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,5120,0.045744889312320285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,2560,0.022310222188631695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,3072,0.03035911255412632
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,4096,0.03458044595188565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,2048,0.017484444710943434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,1024,0.010429333481523724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,65536,0.7885120179918078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,3584,0.0325911111301846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,1536,0.014477334088749357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,1536,0.021964444054497614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,2560,0.027686221732033625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,512,0.006824888702895906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,256,0.005115555392371284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,2048,0.024498666326204937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,1024,0.01872977779971229
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,512,0.01610311037964291
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,768,0.017684444785118103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,768,0.008798222574922774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,64,0.003240888938307762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,32,0.0030826667530669104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,256,0.01442311041884952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,128,0.004007110993067424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,128,0.013548444542619916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,64,0.013157332936922709
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,32,0.013100444442696042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,16384,0.1597635613547431
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,12288,0.12277511755625407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,10240,0.08921955691443549
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,16384,0.1193279955122206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,12288,0.08578311072455512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,8192,0.07293955485026042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,7168,0.06526399983300103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,7168,0.05420355664359199
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,6144,0.055664002895355225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,8192,0.061294224527147084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,10240,0.07426044676038954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,65536,0.5934711032443577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,5120,0.049385776122411094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,6144,0.04783199893103706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,4096,0.040320889817343816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,3584,0.03591288791762458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,3072,0.0210506667693456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,5120,0.039334221018685236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,4096,0.03388177686267429
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,2560,0.019147555033365887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,3584,0.03176888823509216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,3072,0.029320889049106177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,1536,0.013587555951542325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,1024,0.010138666464222802
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,65536,0.7674213515387641
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,1536,0.021406221720907424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,2048,0.01676177812947167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,2048,0.02369955513212416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,2560,0.02664088871743944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,768,0.00850311087237464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,512,0.006697777658700943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,256,0.004948444250557157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,128,0.0037457777394188773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,64,0.003036444385846456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,512,0.015849777393870883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,768,0.01738666660255856
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,32,0.002871111035346985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,256,0.01422755585776435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,128,0.013223110801643796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,64,0.013095999757448832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,1024,0.018364444375038147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,32,0.013011555704805585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,16384,0.10907555288738674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,12288,0.08382399876912434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,16384,0.10740000009536743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,10240,0.07097155518001981
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,12288,0.0788195530573527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,7168,0.05134133497873942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,8192,0.056695110268063016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,10240,0.06831910875108507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,6144,0.04494755466779073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,8192,0.058456891112857394
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,65536,0.5053697692023383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,7168,0.04948622319433424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,5120,0.03847377830081516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,4096,0.02863022353914049
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,6144,0.04349066813786825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,3584,0.019055111540688407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,3072,0.01685422162214915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,2560,0.01828533411026001
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,5120,0.03839200072818332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,3584,0.030639999442630347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,2048,0.015842666228612263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,3072,0.028111110130945843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,65536,0.7480879889594184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,1024,0.008979555633332994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,2560,0.025518221987618342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,1536,0.020285333196322124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,4096,0.033286223808924355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,1536,0.01203733351495531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,768,0.007535999847782983
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,512,0.005991111199061076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,128,0.0036106668412685394
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,768,0.0159599996275372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,2048,0.02313599983851115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,1024,0.01737333337465922
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,256,0.004502222355869082
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,256,0.01332800007528729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,512,0.014700444208251106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,64,0.0028328889360030494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,128,0.01220355596807268
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,32,0.0027057776848475137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,64,0.011896889242861005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,16384,0.08377955357233684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,12288,0.0649048884709676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,32,0.011711999773979187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,10240,0.05486577749252319
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,16384,0.09930666950013901
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,12288,0.07426133420732287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,8192,0.04580622249179416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,7168,0.04240177737341987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,65536,0.37978222635057235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,8192,0.05300177799330818
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,6144,0.03628888726234436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,10240,0.06435288985570271
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,7168,0.04776888754632738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,5120,0.021223111285103693
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,4096,0.017991110682487488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,3584,0.01734044485621982
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,6144,0.0428435570663876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,5120,0.03759200043148465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,3072,0.01726222203837501
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,2560,0.015695111619101632
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,4096,0.032475554280810885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,3072,0.027391110857327778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,1536,0.010286221901575724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,2048,0.013576889203654395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,2560,0.024852444728215534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,3584,0.030066665675905015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,65536,0.6898213492499458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,768,0.006480888773997624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,2048,0.022216889593336318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,1536,0.01961955593691932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,1024,0.016763556334707472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,256,0.004110222061475118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,512,0.014144000079896716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,128,0.0033297776761982176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,1024,0.007780444290902879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,768,0.015471999843915304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,256,0.012633778154850006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,64,0.0026462222966882917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,512,0.005380444642570283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,32,0.0024577778660588795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,128,0.011615999870830111
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,64,0.011362666885058085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,16384,0.05713511175579495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,32,0.011172444456153445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,12288,0.0447599987188975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,16384,0.09415466917885675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,8192,0.022430222895410325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,10240,0.038188444243537054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,65536,0.2717386616600884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,12288,0.07338666915893555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,7168,0.018784888916545443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,6144,0.01947022146648831
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,10240,0.06298222144444783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,5120,0.016600888636377122
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,7168,0.047431111335754395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,8192,0.052427556779649526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,4096,0.014087999860445658
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,6144,0.042544888125525586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,3072,0.01218844453493754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,5120,0.037119110425313316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,3584,0.02940444482697381
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,3584,0.013329777452680798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,3072,0.027136888768937852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,65536,0.6475751135084364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,2048,0.009902222288979424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,4096,0.03211466802491082
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,1536,0.008260444634490544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,1024,0.00638488887084855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,2560,0.0243857784403695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,2560,0.011037333144081963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,768,0.005576000031497743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,2048,0.021736888421906367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,512,0.004583999928500918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,256,0.0037324445115195382
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,1536,0.01923288901646932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,1024,0.01645066671901279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,768,0.014848000473446317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,128,0.003088888815707631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,64,0.0025031111306614345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,512,0.013648889131016202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,32,0.002291555619902081
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,256,0.01219466659757826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,128,0.011298666397730509
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,16384,0.04519911275969612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,64,0.01092266705301073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,32,0.010771555205186209
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,12288,0.035199112362331815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,10240,0.020512888828913372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,16384,0.09399288892745972
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,65536,0.21997066338857016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,8192,0.016543111867374845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,10240,0.0629440016216702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,7168,0.018984888990720112
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,6144,0.017307556337780423
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,12288,0.07327377796173096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,8192,0.052616000175476074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,7168,0.04753333330154419
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,5120,0.014744889405038623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,4096,0.01262399968173769
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,6144,0.04234133495224846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,3584,0.012102222277058495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,3072,0.0111191107167138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,5120,0.03715288970205519
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,2560,0.009860444400045607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,2048,0.009847111172146266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,4096,0.031911111540264554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,3584,0.029449777470694646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,3072,0.02690222197108799
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,1536,0.008209777375062307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,1024,0.0063999998900625445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,2560,0.0244151105483373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,768,0.005578666511509154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,2048,0.02183466653029124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,1536,0.019155555301242404
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,512,0.004590222405062782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,256,0.0037377778854635027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,1024,0.01625777781009674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,128,0.002946666752298673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,65536,0.6603288650512695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,768,0.01480977733929952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,512,0.01351999988158544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,64,0.002327999927931362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,256,0.012156444291273752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,128,0.011222222612963783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,64,0.010971555279360877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,16384,0.024326221810446844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,32,0.010748444332016839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,12288,0.01928711103068458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,32,0.0021520000365045336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,10240,0.016917333006858826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,65536,0.1535253392325507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,8192,0.014361777239375643
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,7168,0.01316177762216992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,16384,0.09401244587368435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,12288,0.07318666908476087
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,6144,0.012058667010731168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,10240,0.06311021910773383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,5120,0.011747555600272285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,8192,0.052481777138180204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,7168,0.04768800073199802
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,4096,0.010128888818952773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,3584,0.009510222408506606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,6144,0.04256355431344774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,3072,0.00868800034125646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,2560,0.00794222205877304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,4096,0.031903111272388034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,5120,0.03709955679045783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,2048,0.0070257774657673305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,3584,0.02959111001756456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,3072,0.02686044408215417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,1536,0.00923466682434082
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,2560,0.02441066669093238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,1024,0.006127999888526068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,768,0.00535733335547977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,2048,0.021743999587164983
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,512,0.004409777621428172
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,65536,0.6664355595906576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,1536,0.019170666734377544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,1024,0.016229333149062265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,256,0.0032604444358083936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,768,0.014948444234000312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,64,0.0021404443929592767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,512,0.01349066694577535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,128,0.0025920000755124623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,32,0.0023502223193645477
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,256,0.012171555724408893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,128,0.011202666494581433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,16384,0.016495111915800307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,64,0.01088444474670622
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,65536,0.05522222320238749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,12288,0.0136817776494556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,32,0.010724444356229572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,10240,0.012039999994966718
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,8192,0.01020088874631458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,7168,0.009631111390060848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,16384,0.09416444434059991
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,6144,0.008886222210195329
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,10240,0.06296177705128987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,8192,0.052453332477145724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,12288,0.07342933283911811
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,5120,0.007986666427718269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,7168,0.047536889712015785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,4096,0.0070364442136552595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,3584,0.006784889019197888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,3072,0.006351999938488007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,6144,0.04266666703754001
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,2560,0.006015111174848344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,5120,0.03700977894994948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,3584,0.029510223203235205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,4096,0.031889776388804116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,2048,0.005489777773618698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,3072,0.026829333768950567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,1536,0.008906666603353288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,2560,0.024345777100986902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,1024,0.006518222391605377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,768,0.005362666729423735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,2048,0.021503999829292297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,512,0.004159111115667555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,1536,0.019131556153297424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,1024,0.016031111280123394
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,65536,0.6710782051086426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,256,0.0030088888274298776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,768,0.0147724449634552
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,128,0.002377777877781126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,64,0.001992888955606355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,32,0.0021911110315057966
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,256,0.012089778151777057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,65536,0.02940622303220961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,512,0.01350933313369751
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,64,0.010842666857772403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,32,0.0107360002067354
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,16384,0.010474666953086853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,128,0.011163555913501315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,12288,0.00889244427283605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,10240,0.008040000167157915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,8192,0.007252444823582967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,7168,0.006839111033413146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,12288,0.07332355446285672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,6144,0.006425777657164468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,10240,0.0628933310508728
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,16384,0.09382400247785781
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,8192,0.052467554807662964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,5120,0.005960888746711943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,4096,0.005505777895450592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,3584,0.00564177789621883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,7168,0.04786133435037401
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,6144,0.04235022266705831
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,3072,0.005141333159473208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,5120,0.03698222173584832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,4096,0.031806220610936485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,2560,0.0064088888466358185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,2048,0.005710222240951326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,3584,0.029550221231248643
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,1536,0.0047182221379545005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,3072,0.02719288898838891
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,1024,0.005807999935415056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,768,0.0048142220411035745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,2560,0.024286222126748826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,2048,0.021631111701329548
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,1536,0.018921777606010437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,512,0.0038284444146686127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,65536,0.6567360030280219
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,256,0.0028853333658642242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,1024,0.016002666619088914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,128,0.0023804443577925363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,64,0.0021831111775504216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,768,0.014834667245546976
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,32,0.0021173333128293357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,512,0.013433777623706393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,256,0.011957333319716983
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,65536,0.03021777669588725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,16384,0.010975999964608086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,64,0.010803555448849997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,128,0.011212444139851464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,32,0.010732444624106089
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,12288,0.009298666483826106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,10240,0.008565333154466417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,8192,0.007507555186748505
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,7168,0.007120889094140794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,16384,0.09328889184527928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,12288,0.07305510838826497
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,6144,0.006702222343948152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,10240,0.06281866629918416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,8192,0.05236088898446825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,7168,0.04721155431535509
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,5120,0.0063680000603199005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,4096,0.005701333284378052
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,3584,0.005620444400442972
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,6144,0.042043556769688926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,3072,0.005154666801293691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,5120,0.03680800067053901
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,4096,0.03170933326085409
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,2560,0.005013333426581489
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,3584,0.029166221618652344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,2048,0.004519999855094486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,1536,0.004132444245947732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,3072,0.026615111364258662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,1024,0.005564444594913059
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,2560,0.024068444967269897
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,768,0.004648000001907349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,2048,0.02146133283774058
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,512,0.003687111039956411
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,1536,0.018784888916545443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,256,0.0027644443843099806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,1024,0.015878222054905362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,128,0.002266666748457485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,768,0.014492443866199918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,65536,0.6579795413547093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,64,0.0021564445147911706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,512,0.013192888763215808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,32,0.0021226666867733
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,256,0.011909333368142446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,65536,0.017638223038779367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,128,0.01108444482088089
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,16384,0.007829333345095316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,64,0.010757333702511258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,12288,0.006952889263629913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,10240,0.00665511108107037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,32,0.010679110884666443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,8192,0.006051555689838197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,16384,0.0928257770008511
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,7168,0.00574311117331187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,12288,0.07252889209323458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,10240,0.06222222248713175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,6144,0.0052284445199701525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,8192,0.05235111051135593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,5120,0.005092444519201915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,4096,0.004976888911591636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,7168,0.04683466752370199
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,3584,0.004869333157936732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,6144,0.04187644521395365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,5120,0.03657333387268914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,4096,0.031397332747777305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,3072,0.004497777670621872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,2560,0.004695999953481886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,2048,0.003970666478077571
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,1536,0.0038293335172865125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,3584,0.028845333390765723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,3072,0.026455110973782007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,2560,0.023754666248957317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,1024,0.0035582222044467926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,2048,0.021316443880399067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,768,0.004518222063779831
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,512,0.0036071112586392295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,65536,0.6494320233662924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,1536,0.01869422197341919
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,256,0.0026737778551048706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,1024,0.01575199928548601
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,768,0.014428445034556918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,64,0.0021377777059872947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,128,0.0022453332526816265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,512,0.013211555778980255
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,256,0.011876444849703046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,32,0.0021093332519133887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,128,0.010964444114102257
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,32,0.01068800025516086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,64,0.010701333483060202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,10240,1.4195511076185439
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,12288,1.6924453311496312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,8192,1.8142532772488063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,16384,2.1952667236328125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,10240,2.216549343532986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,7168,0.9199822743733724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,8192,1.0983787112765842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,12288,2.8969679938422312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,7168,1.5714391072591145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,6144,0.816399097442627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,6144,1.3939652972751195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,5120,1.136305809020996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,16384,3.502457724677192
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,4096,0.5206258032057021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,4096,0.9246070649888781
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,5120,0.6835315492418078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,3584,0.46086491478814023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,3584,0.7947493129306369
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,2560,0.5761377546522353
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,2560,0.3746719890170627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,1536,0.36536889606051975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,2048,0.46896176868014866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,3072,0.43307288487752277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,2048,0.3062613275316027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,1024,0.26118400361802846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,3072,0.744143115149604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,512,0.12877866956922743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,1024,0.1885111067030165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,512,0.11869866318172878
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,128,0.03870044483078851
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,768,0.21642577648162842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,256,0.10658844312032063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,1536,0.24596089786953398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,32,0.02164977788925171
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,64,0.02218311031659444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,256,0.07151377863354154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,64,0.09452533059649998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,32,0.09344800313313802
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,128,0.10531644688712226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,768,0.14954577551947698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,10240,0.40773601002163357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,12288,0.5143724547492133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,10240,0.6127511130438911
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,16384,0.6652773221333822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,16384,0.8769333097669813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,12288,0.8072666592068143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,8192,0.2935946782430013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,6144,0.3419564565022786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,6144,0.21799911393059623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,8192,0.46535375383165145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,7168,0.2738862302568224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,7168,0.41133777300516766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,5120,0.2805742157830132
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,5120,0.1810826725429959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,4096,0.2299706670973036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,3584,0.127475553088718
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,4096,0.15502133634355333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,3584,0.21874577469295928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,2560,0.1362124416563246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,3072,0.19098933537801108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,2048,0.1076586643854777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,65536,2.0653849707709417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,2048,0.08801600005891587
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,3072,0.1191662285063002
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,1536,0.06937777996063232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,1536,0.09198044406043158
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,1024,0.05621066358354357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,2560,0.1036977767944336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,768,0.04801511102252536
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,512,0.022625777456495497
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,256,0.01129155523247189
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,128,0.008492444124486711
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,1024,0.05756622552871704
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,768,0.044623110029432506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,32,0.0053093334039052325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,64,0.006881777611043718
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,256,0.03609155615170797
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,128,0.03300977746645609
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,512,0.040707555082109236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,32,0.031397332747777305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,64,0.03160355488459269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,12288,0.5513680246141222
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,12288,0.4034106731414795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,16384,0.5144844584994847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,16384,0.6781386799282498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,65536,3.432754728529188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,10240,0.41882843441433376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,8192,0.24346666865878633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,8192,0.3711511029137506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,7168,0.308676454755995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,7168,0.2109422286351522
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,10240,0.3055093288421631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,6144,0.2714995543162028
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,5120,0.22157866424984404
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,4096,0.17720533741845024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,6144,0.18337689505683052
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,3584,0.15257599618699816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,65536,1.7059955596923828
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,5120,0.15015378263261583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,3072,0.12884444660610622
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,3584,0.10386844476064046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,4096,0.12176356050703262
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,2560,0.08173688915040758
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,2048,0.08765333228641087
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,3072,0.0966799987687005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,1536,0.06781511174307929
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,2048,0.07077599896325006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,1024,0.04750666684574551
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,768,0.036701334847344294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,512,0.017809778451919556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,1024,0.044327110052108765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,128,0.008143111235565608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,2560,0.11708266205257839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,1536,0.06253333224190606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,256,0.011783999701340994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,64,0.006334222025341458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,256,0.03215466605292426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,768,0.04020444552103678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,512,0.036483556032180786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,32,0.00461777796347936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,64,0.028148445818159316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,128,0.0295724438296424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,32,0.028205331828859117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,65536,2.626352098253038
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,12288,0.35574666659037274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,10240,0.28739110628763836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,16384,0.46195464664035374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,12288,0.44795555538601345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,16384,0.6061689058939616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,10240,0.3832826614379883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,8192,0.286993768480089
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,8192,0.23310311635335287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,7168,0.20155021879408094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,7168,0.3005288971794976
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,6144,0.26406221919589573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,6144,0.17143910461001924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,4096,0.16920444700453016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,5120,0.2250657876332601
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,3584,0.11299022038777669
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,5120,0.1462791098488702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,3584,0.10727200243208145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,4096,0.11928799417283799
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,3072,0.09113422367307876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,3072,0.10952177974912856
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,2560,0.08405422502093846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,1536,0.05120977759361267
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,2560,0.09154222408930461
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,2048,0.06869066423839994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,1024,0.039834668238957725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,65536,1.6804996066623266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,768,0.01826844447188907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,1536,0.057405332724253334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,2048,0.07441777653164335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,256,0.009135111338562435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,128,0.006888888776302338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,768,0.03864799936612447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,256,0.031086222993002996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,64,0.005664888769388199
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,1024,0.04275733232498169
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,512,0.013245333400037555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,32,0.004193777839342753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,128,0.02827822168668111
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,64,0.026749332745869953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,512,0.03534311056137085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,32,0.02693333393997616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,65536,2.1897591484917536
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,10240,0.2788124349382189
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,12288,0.35869420899285215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,16384,0.4512622091505263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,12288,0.28408800231085884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,16384,0.37175644768608945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,8192,0.2279120021396213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,8192,0.1597599983215332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,10240,0.22425333658854166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,7168,0.1324266725116306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,7168,0.21511467297871908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,5120,0.14787199762132433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,6144,0.19003732999165854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,6144,0.11441066530015732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,4096,0.10902755790286595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,5120,0.09070933527416652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,3584,0.09385599692662557
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,65536,1.1794231202867296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,3072,0.08155822091632418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,3584,0.06790488958358765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,4096,0.08122577932145861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,2560,0.06885511345333524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,3072,0.06126488579644097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,1024,0.020875554945733812
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,2560,0.05421511001057095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,1536,0.047351999415291675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,2048,0.04567911227544149
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,65536,1.7415324317084417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,2048,0.062053336037529834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,768,0.027297778262032404
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,1024,0.028844445943832397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,512,0.024666666984558105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,256,0.007231999602582719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,768,0.01516355574131012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,1536,0.03366577625274658
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,128,0.0053688887920644544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,64,0.004446222136418025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,32,0.0036959999965296853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,256,0.022873777482244704
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,32,0.02000266644689772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,512,0.011060444845093621
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,64,0.0199644449684355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,128,0.02083822256988949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,16384,0.2957333458794488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,10240,0.1780595514509413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,12288,0.22117243872748482
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,12288,0.3447564442952474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,16384,0.4301519923739963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,10240,0.26857156223720974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,8192,0.13366756174299452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,8192,0.2021039989259508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,6144,0.14644532733493382
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,7168,0.11936088403065999
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,7168,0.17375644048055014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,5120,0.11634133921729194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,4096,0.09689422448476155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,6144,0.1027911106745402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,5120,0.08156355884340075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,3584,0.08312444554434882
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,3072,0.07240711318122016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,65536,1.0810480117797852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,3584,0.06233244472079807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,1536,0.040927112102508545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,4096,0.0736337767706977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,2560,0.04969155457284716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,2560,0.06771555874082777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,3072,0.05938666396670871
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,768,0.015096000499195524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,1024,0.0192577772670322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,1536,0.0317813323603736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,512,0.010919111470381418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,2048,0.05507111218240526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,256,0.007238222493065729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,65536,1.5569288465711806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,2048,0.03994755612479316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,128,0.0052693333062860705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,768,0.025754666990704004
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,512,0.023352000448438857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,256,0.021454221672481958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,32,0.0033688888781600525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,128,0.019334221879641216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,1024,0.02734222180313534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,32,0.018981332580248516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,64,0.019474666979577806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,64,0.00434666665063964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,16384,0.2569199932946099
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,16384,0.3432782226138645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,12288,0.27030666669209796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,12288,0.19133955902523467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,10240,0.20985066890716553
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,10240,0.15578667322794595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,8192,0.17358044783274332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,7168,0.14383199479844835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,6144,0.12428532706366645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,7168,0.09647733635372585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,5120,0.10107111268573338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,8192,0.110953774717119
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,6144,0.0924346645673116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,5120,0.07268889082802667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,3584,0.07195999887254503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,4096,0.08688888947168986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,65536,0.9684862560696073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,2560,0.05273422267701891
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,3072,0.05023555623160469
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,4096,0.0611635579003228
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,2048,0.04392355680465698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,3584,0.059879998366038
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,1536,0.038352890147103205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,3072,0.0719368921385871
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,2560,0.041301333241992526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,768,0.014879110786649915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,1024,0.018008000320858426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,65536,1.3232559627956815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,512,0.010699555277824402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,1024,0.026563555002212524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,1536,0.030384888251622517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,2048,0.03538399934768677
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,128,0.0052177777720822235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,512,0.02202933364444309
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,768,0.02436622149414486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,64,0.0041457778877682155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,256,0.02019466625319587
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,256,0.007034666836261749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,128,0.018431110514534842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,32,0.003096888876623578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,64,0.01833866702185737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,32,0.01793066660563151
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,12288,0.15122666623857287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,16384,0.22075377570258248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,12288,0.22844799359639487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,10240,0.19224177466498482
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,16384,0.31106577979193795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,10240,0.11440089013841416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,8192,0.09494488769107395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,7168,0.11253510581122504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,8192,0.14393777317470974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,7168,0.08924977646933661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,6144,0.10011821985244751
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,5120,0.06398133436838786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,5120,0.08310133218765259
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,3584,0.0575795571009318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,4096,0.05432088838683235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,6144,0.07292711072497897
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,4096,0.06912799676259358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,65536,0.8574079937405057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,3584,0.04908711049291822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,3072,0.050311999188529126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,2560,0.041943109697765775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,2048,0.03452355662981669
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,65536,1.1006559795803494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,2048,0.034112887250052557
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,1024,0.011766222616036734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,3072,0.042372445265452065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,1536,0.030227555168999568
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,1536,0.0166302224000295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,512,0.007546666595670912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,768,0.009769777456919352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,2560,0.03827733463711209
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,256,0.005346666607591841
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,128,0.0041866666740841335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,64,0.003588444242874781
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,256,0.019448889626397025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,512,0.021707556313938562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,32,0.0029146667155954572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,128,0.018000000052981906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,64,0.017500445246696472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,1024,0.025973333252800837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,768,0.023775999744733173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,32,0.017684444785118103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,16384,0.2403857707977295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,10240,0.14311110973358154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,16384,0.15798489252726236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,12288,0.10520888699425592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,12288,0.20319822099473742
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,10240,0.08777333630455865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,7168,0.09815555810928345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,8192,0.07256533039940728
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,5120,0.07151910993787977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,6144,0.0947635571161906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,7168,0.06447199980417888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,6144,0.05712711148791843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,8192,0.12427643934885661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,4096,0.05936444467968411
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,5120,0.050181332561704844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,3584,0.052335109975602895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,65536,0.7184791035122342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,2560,0.03371555606524149
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,3584,0.03169155451986525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,3072,0.04032266802257962
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,65536,0.8969839943779839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,2048,0.018183999591403537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,1024,0.010467555787828235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,2560,0.026798221800062392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,1536,0.014560000763999091
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,2048,0.023448000351587932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,768,0.00885511106914944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,1536,0.021327111456129286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,512,0.006888888776302338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,4096,0.04105510976579454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,1024,0.01816444430086348
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,128,0.003950222084919612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,256,0.005085333353943295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,768,0.01741155485312144
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,256,0.014471999473041959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,32,0.003097777772280905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,64,0.0032302221904198327
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,64,0.013342222405804528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,3072,0.029169778029123943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,32,0.01295733369059033
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,128,0.013263111313184103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,512,0.01570311188697815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,16384,0.13533688916100398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,16384,0.2158177826139662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,12288,0.15636000368330213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,10240,0.1358097791671753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,8192,0.06754666566848755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,12288,0.10221866766611735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,10240,0.0816391110420227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,8192,0.09891021913952297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,65536,0.6740818023681641
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,6144,0.07598844501707289
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,7168,0.0969582200050354
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,5120,0.06500177913241915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,6144,0.053209778335359365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,7168,0.06016088856591118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,65536,0.7924613422817655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,3584,0.04638222191068861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,3584,0.030563556485705908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,4096,0.052311999930275806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,2560,0.02618933386272854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,3072,0.02829155657026503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,3072,0.03561777869860331
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,5120,0.046907554070154824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,2048,0.01700622174474928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,4096,0.03534577621353997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,1024,0.0101742222905159
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,768,0.008551110823949179
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,2560,0.025751110580232408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,1536,0.020861332615216572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,512,0.006712889091836081
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,1024,0.017815111411942374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,768,0.017127111554145813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,256,0.005008000052637524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,1536,0.013602666556835175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,512,0.015538665983412
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,128,0.0037573331760035623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,64,0.0031155554784668815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,2048,0.02336799932850732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,256,0.01426755554146237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,32,0.0029644444584846497
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,128,0.013533333109484779
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,64,0.013088888592190213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,32,0.012683555483818054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,12288,0.12995289431677923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,16384,0.11752622657352024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,16384,0.18899733490414092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,10240,0.10533333486980861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,8192,0.08832444747289021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,10240,0.07560533285140991
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,7168,0.0788408915201823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,8192,0.06269688738716973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,6144,0.0678044425116645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,7168,0.0557840002907647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,5120,0.05741688940260145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,4096,0.047687109973695546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,65536,0.6932409074571398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,12288,0.08815111054314508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,65536,0.6201333469814724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,5120,0.04134577843878005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,3072,0.030718220604790583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,6144,0.04961511161592272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,3584,0.029896888468000624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,3584,0.04045244389110141
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,2560,0.020457777712080214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,2560,0.02548622257179684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,2048,0.016982222596804302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,1536,0.013430222041077085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,1024,0.010039110978444418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,3072,0.02786133355564541
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,4096,0.03246488836076524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,768,0.008396444221337637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,2048,0.02271466619438595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,256,0.004892444445027245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,1536,0.020367999871571858
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,1024,0.01791733337773217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,128,0.003721777763631609
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,768,0.0162773331006368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,64,0.002990222225586573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,512,0.015415999624464246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,32,0.0027884443600972495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,128,0.012894222305880653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,512,0.006637333167923822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,64,0.012821333275900947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,32,0.012355555262830524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,256,0.013996443814701505
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,16384,0.1471831136279636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,10240,0.08668444554011027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,12288,0.07995733287599352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,12288,0.10469777716530694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,16384,0.10627733336554633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,8192,0.07136888636483087
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,7168,0.06497333447138469
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,10240,0.06894489129384358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,6144,0.055161777469846934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,8192,0.057793776194254555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,7168,0.0511813329325782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,5120,0.04834666517045763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,4096,0.040241777896881104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,65536,0.5906951162550185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,5120,0.036080890231662326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,65536,0.5886578030056423
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,3584,0.02907022171550327
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,6144,0.04432000054253472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,4096,0.03140711122088962
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,3072,0.026760000321600173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,3072,0.019689778486887615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,3584,0.0347724457581838
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,2048,0.015058666467666626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,2560,0.017475555340449016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,1536,0.013253333667914072
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,1024,0.008800889054934183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,768,0.007377777662542131
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,2560,0.024308444725142583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,512,0.0058853332367208265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,2048,0.021866665946112737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,256,0.004416000097990036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,1024,0.016583111551072862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,1536,0.019297776950730216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,768,0.01556799974706438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,128,0.003549333247873518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,64,0.0027928888383838865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,32,0.0026542221506436667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,256,0.012853333519564735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,128,0.011974222130245633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,512,0.014262222581439547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,64,0.011709333293967776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,32,0.011657777759763928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,12288,0.0830871131685045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,16384,0.11008533504274155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,10240,0.07061600022845797
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,16384,0.09762222237057155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,12288,0.07263021998935275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,8192,0.05819466378953722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,7168,0.0513937771320343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,10240,0.06415644619199964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,8192,0.053377777338027954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,6144,0.045243554645114474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,7168,0.044963555203543774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,65536,0.48689513736301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,5120,0.03907733493381076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,6144,0.04028355413013034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,4096,0.022895110978020564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,3584,0.01880088945229848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,65536,0.5621040132310655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,3072,0.017317333155208163
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,4096,0.030928889910380047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,2048,0.01384622189733717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,2560,0.015414221419228448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,5120,0.03552800085809495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,1536,0.010187555518415239
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,1024,0.00778399987353219
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,2560,0.023804444405767653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,2048,0.021350221501456365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,768,0.006518222391605377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,3584,0.028581334484948054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,512,0.005366222312053044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,1024,0.016315556234783597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,3072,0.026154667139053345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,256,0.004655111167165968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,768,0.0151582227812873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,128,0.0033235556135574975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,64,0.002612444468670421
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,512,0.013764444324705334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,32,0.002444444431198968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,1536,0.019001776973406475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,256,0.012576888832781049
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,128,0.011424000064531961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,64,0.011247999966144562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,32,0.011186666786670685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,12288,0.06413600179884169
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,16384,0.08282311095131768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,10240,0.05481688843833076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,12288,0.06886666350894503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,16384,0.09071377913157146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,8192,0.04534399840566847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,7168,0.03785599933730231
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,65536,0.37290223439534503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,10240,0.05959910816616482
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,6144,0.033039109574423894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,8192,0.049675554037094116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,5120,0.018762666318151686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,7168,0.04465688930617439
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,4096,0.016161777906947665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,6144,0.03986488779385885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,65536,0.5200586848788792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,5120,0.03513600097762214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,3584,0.014909333652920194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,2560,0.012335110869672565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,2048,0.010776888993051318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,4096,0.03068000078201294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,3072,0.025828444295459326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,3584,0.0281644430425432
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,1536,0.008857777549160851
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,3072,0.01346933344999949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,1024,0.006706666615274217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,768,0.005843555761708154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,512,0.004902222090297275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,1536,0.01869066721863217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,1024,0.015803555647532146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,256,0.0038933331767717996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,2048,0.02087822225358751
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,768,0.014679999815093147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,2560,0.023374223046832617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,128,0.0030577776746617425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,64,0.0024764444679021835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,32,0.0023431111541059283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,512,0.013559111290507846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,256,0.012267555627557965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,128,0.011123555401961008
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,64,0.010993777877754636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,32,0.010783111055692038
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,12288,0.04458933406405979
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,16384,0.05734133058124118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,10240,0.03597066799799601
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,8192,0.01997244523631202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,16384,0.08750310871336196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,65536,0.26107909944322377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,12288,0.06839022371504042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,10240,0.05877600113550822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,7168,0.01793955597612593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,6144,0.016113777955373127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,8192,0.04939200149642097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,5120,0.013905778527259827
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,7168,0.04477511180771721
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,3584,0.011576889289749993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,6144,0.03972888986269633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,3072,0.010575999816258749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,4096,0.011888888974984487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,4096,0.030419554975297716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,2560,0.009452444811662039
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,65536,0.47911908891465926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,2048,0.009836444424258338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,5120,0.03521777855025397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,3584,0.028194665908813477
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,3072,0.025751110580232408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,2560,0.023237332701683044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,1024,0.0063884444534778595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,1536,0.008185777399275038
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,768,0.0055440002017551
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,2048,0.020792000823550753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,1536,0.018423111902342904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,512,0.004569777597983678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,256,0.0037137779096762338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,1024,0.015811555915408664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,128,0.0029288888391521242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,64,0.0022400000856982335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,768,0.014577777849303352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,32,0.0021280000607172647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,256,0.01198933356338077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,128,0.011185777684052786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,64,0.010944888823562197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,512,0.013341333303186627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,16384,0.04254133171505398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,12288,0.03314666615592109
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,10240,0.01812533371978336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,65536,0.20561956034766304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,12288,0.06832533412509494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,8192,0.015258666541841296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,32,0.010760889285140567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,16384,0.08762400017844306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,6144,0.013912889692518445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,7168,0.014774221513006421
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,10240,0.058861335118611656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,5120,0.012282667060693106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,8192,0.049441777997546725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,4096,0.010867555936177572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,65536,0.472492429945204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,3584,0.010085333552625444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,6144,0.039919998910692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,4096,0.030497776137457952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,3072,0.009377777576446533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,7168,0.04455022348297966
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,2048,0.009568000005351173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,2560,0.01051111105415556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,3584,0.028177777926127117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,5120,0.03512977891498142
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,1536,0.008011555506123437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,2560,0.023218666513760883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,1024,0.006233777850866318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,2048,0.020844444632530212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,768,0.005399111244413588
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,512,0.004421333471934001
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,256,0.0035004446076022256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,1536,0.018530666828155518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,3072,0.025628444221284654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,1024,0.01564800077014499
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,128,0.0027831111931138565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,768,0.014496000276671516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,512,0.013296888934241401
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,64,0.002198222196764416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,256,0.012103999654452005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,32,0.0025475554996066624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,128,0.011141333315107556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,32,0.010760889285140567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,64,0.010865777730941772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,16384,0.025928889711697895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,12288,0.015635554989178974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,65536,0.1252222193611993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,10240,0.013765333427323235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,16384,0.08728710810343425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,8192,0.011909333368142446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,7168,0.011400000088744693
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,6144,0.011384000380833944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,10240,0.058840003278520375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,12288,0.06865600082609388
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,8192,0.049286223120159574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,5120,0.010035555395815108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,7168,0.04456622070736355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,4096,0.008658666577604082
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,6144,0.03980177640914917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,3072,0.007725333174069722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,3584,0.008288000192907121
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,4096,0.030304001437293157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,5120,0.03520800007714166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,2560,0.00700444479783376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,3584,0.028001778655582007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,65536,0.4536426862080892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,3072,0.025711110896534387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,1536,0.009024889104896123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,2048,0.0063928887248039246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,1024,0.006667555620272954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,2560,0.023312888211674158
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,768,0.005483555710977978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,2048,0.020762667059898376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,1024,0.015571556157535978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,512,0.004293333325121137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,1536,0.018273777431911893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,256,0.0031351111829280853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,128,0.00249422217408816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,64,0.0020924444413847392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,768,0.014460444450378418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,32,0.0022862222459581164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,256,0.012259555359681448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,512,0.01332355539004008
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,64,0.010842666857772403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,128,0.011086222198274402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,65536,0.05490488807360331
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,32,0.010746666954623329
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,12288,0.01148799972401725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,10240,0.010312889185216691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,8192,0.008732444710201686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,16384,0.013844444519943662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,7168,0.008419555922349295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,10240,0.05875200033187866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,12288,0.06828888919618395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,16384,0.08745688862270778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,6144,0.007606222397751278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,8192,0.04923644330766466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,5120,0.0070266665683852295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,4096,0.006342222293217977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,3584,0.006061333335108227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,7168,0.044716444280412465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,6144,0.03976088762283325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,3072,0.005990222096443176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,65536,0.44163555569118923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,5120,0.035178667969173856
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,4096,0.03030844529469808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,2560,0.005484444399674733
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,3584,0.02826666831970215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,3072,0.02569688856601715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,2048,0.00499911109606425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,1536,0.008791999684439765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,1024,0.006446222050322427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,2560,0.023129777775870428
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,768,0.0052595556610160405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,1536,0.018279999494552612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,512,0.004096888833575779
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,2048,0.020821332931518555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,256,0.002955555501911375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,128,0.0023831110447645187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,1024,0.015609777635998197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,768,0.014475555883513557
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,64,0.0022257777551809945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,512,0.01313866674900055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,32,0.002193777718477779
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,256,0.01186044431395001
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,65536,0.028096887800428603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,128,0.01109777804878023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,16384,0.009348444640636444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,64,0.010789333118332757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,12288,0.008099555969238281
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,32,0.010753778119881948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,10240,0.0075333333677715724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,8192,0.006700444552633498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,7168,0.009130666653315226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,16384,0.08712889088524713
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,12288,0.06851910882525973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,10240,0.059119999408721924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,6144,0.006036444256703059
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,8192,0.04920533299446106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,7168,0.0446684459845225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,5120,0.005592888842026393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,4096,0.005547555370463266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,3584,0.0069271110826068455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,6144,0.03985155622164408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,5120,0.03503377901183234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,3072,0.006365333166387346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,4096,0.03037244412634108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,65536,0.4370311101277669
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,3584,0.027968890137142603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,2560,0.005853333406978183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,2048,0.005647111270162795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,1024,0.005695111221737332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,1536,0.0046657779150538975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,3072,0.025800888737042744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,2560,0.023230221536424425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,768,0.004722666823201709
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,2048,0.02069511181778378
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,512,0.0037724446091387006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,1024,0.01556711064444648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,256,0.002830222249031067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,1536,0.01822933389080895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,128,0.002334222197532654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,768,0.01442488862408532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,64,0.00217333332531982
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,512,0.013205333716339536
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,32,0.0021457777669032416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,256,0.011876444849703046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,128,0.010999999940395355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,16384,0.009561777942710454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,65536,0.024167999625205994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,64,0.010834666589895884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,12288,0.008419555922349295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,32,0.010682666467295753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,10240,0.007823111282454597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,8192,0.00702311098575592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,7168,0.007100444700982835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,16384,0.08707111411624485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,6144,0.006512889017661412
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,10240,0.058449778291914195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,12288,0.06815822256935967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,8192,0.04911199874348111
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,5120,0.005973333285914526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,7168,0.044292443328433566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,4096,0.005398222141795688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,3584,0.0053102220926019884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,6144,0.03951288925276862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,3072,0.005214222189452913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,2560,0.004755555755562252
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,5120,0.034801777866151594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,4096,0.030093332131703694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,3584,0.02769600020514594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,2048,0.004641777939266629
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,65536,0.41050132115681964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,1536,0.004044444610675176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,3072,0.025360888904995386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,1024,0.005504888792832692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,768,0.004559111264016894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,2560,0.023043556345833674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,512,0.0036346668170558084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,2048,0.020432889461517334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,1536,0.018013333280881245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,256,0.0027297778675953546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,768,0.014196443888876172
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,1024,0.015378667248619927
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,128,0.0022524444179402455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,512,0.012905778156386482
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,64,0.002161777681774563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,32,0.0021173333128293357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,256,0.01163644426398807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,128,0.010896888871987661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,65536,0.014935111006100973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,16384,0.007335999773608313
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,64,0.010799110763602786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,32,0.0107360002067354
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,12288,0.006842666616042455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,10240,0.006081777728266186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,8192,0.006197333335876465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,7168,0.005508444375462002
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,16384,0.08617599805196126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,12288,0.06720266739527385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,6144,0.005481777919663324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,10240,0.057892441749572754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,8192,0.048581331968307495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,5120,0.005024000174469418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,7168,0.04417511158519321
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,4096,0.004876444323195351
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,3584,0.004657777647177379
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,6144,0.03906311260329352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,3072,0.004359111189842224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,5120,0.03456711106830173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,2560,0.0041039999988343985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,2048,0.0038311113086011675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,4096,0.029743111795849268
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,3072,0.025198222862349615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,3584,0.027570666538344488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,1536,0.003748444633351432
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,65536,0.41563823488023544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,2560,0.02278222143650055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,1024,0.003536000019974179
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,2048,0.020410666863123577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,512,0.003572444534964032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,1536,0.017934223016103108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,768,0.004497777670621872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,1024,0.01531644496652815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,256,0.002805333377586471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,128,0.0022453332526816265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,768,0.014096000128322177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,64,0.002163555473089218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,512,0.012967111335860359
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,32,0.002124444478087955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,256,0.011588444312413534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,128,0.010969777901967367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,64,0.010874666273593903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,32,0.01070666644308302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,10240,1.4004738065931532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,12288,1.674226654900445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,8192,1.806372536553277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,16384,2.171855926513672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,10240,2.206261316935221
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,8192,1.0781164169311523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,12288,2.884515550401476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,7168,0.906958262125651
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,6144,0.7568782170613607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,7168,1.558062235514323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,16384,3.494098663330078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,5120,1.230046272277832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,6144,1.3797839482625325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,5120,0.6644026438395182
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,4096,0.9116995069715711
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,3584,0.7842497825622559
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,4096,0.538823127746582
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,3072,0.39505865838792587
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,2560,0.5677093399895562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,3584,0.4799991183810764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,2048,0.46089601516723633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,3072,0.7348684204949273
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,2560,0.359935998916626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,1536,0.3821840021345351
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,1024,0.25376444392734104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,2048,0.30050399568345815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,512,0.11487555503845215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,768,0.20968088838789198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,256,0.06597155332565308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,1024,0.18190222316318086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,128,0.03597955571280585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,1536,0.24244178666008842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,512,0.1227102279663086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,256,0.10283199946085612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,128,0.10114666488435532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,32,0.01851733360025618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,32,0.09279733233981663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,64,0.09337066941791111
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,768,0.14391556051042345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,64,0.019898666275872123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,16384,0.5922604666815864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,12288,0.49875556098090273
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,10240,0.5933315489027235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,12288,0.7277404467264811
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,10240,0.40304266081915957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,16384,0.9573040008544922
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,8192,0.44399023056030273
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,8192,0.28917333814832896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,7168,0.25883732901679146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,7168,0.418412446975708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,6144,0.37317244211832684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,5120,0.27947378158569336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,6144,0.22463022338019478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,5120,0.18409155474768746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,4096,0.2289093335469564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,4096,0.15198222796122232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,3072,0.1883066627714369
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,3072,0.11038755708270603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,65536,1.9955323537190754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,3584,0.21594044897291395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,2560,0.13653955194685194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,2048,0.12059466044108073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,3584,0.13395378324720594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,2048,0.08181422286563449
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,1024,0.056040002240075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,1536,0.08207466867234972
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,768,0.045984887414508395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,512,0.02516799999607934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,1024,0.05739821990331014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,1536,0.07208089033762614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,768,0.0437466667758094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,2560,0.10192266437742446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,256,0.01295111080010732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,64,0.006773333168692059
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,512,0.03958044449488322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,128,0.008433777425024245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,128,0.032794667614830866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,32,0.004912000149488449
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,32,0.03111999895837572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,64,0.03163466519779629
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,256,0.0358951124880049
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,16384,0.4574337535434299
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,12288,0.5454257859124078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,12288,0.3498968813154433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,16384,0.7297191090053983
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,10240,0.2948977682325575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,65536,3.3961181640625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,10240,0.41542310184902614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,8192,0.3364293310377333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,8192,0.23796621958414713
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,7168,0.19933066103193495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,7168,0.29423022270202637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,6144,0.26106754938761395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,6144,0.16828889316982695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,65536,1.6355031331380208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,5120,0.21507644653320312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,4096,0.1730017794503106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,5120,0.13776089085472956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,3584,0.14434666103786892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,3072,0.12372533480326335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,4096,0.1195217768351237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,3584,0.10865866475635105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,2560,0.11812355783250596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,2560,0.08080355326334636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,3072,0.09678488969802856
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,1024,0.047445333666271634
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,1536,0.07283377647399902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,2048,0.07303644551171197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,2048,0.09314933088090684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,768,0.03612533211708069
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,1536,0.061667554908328585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,256,0.01165422217713462
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,512,0.01704266667366028
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,1024,0.04444266690148247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,128,0.00792266676823298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,768,0.040064887868033514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,512,0.03624177641338772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,65536,2.5641422271728516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,64,0.006048000107208888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,128,0.029305779271655615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,256,0.0321706665886773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,32,0.004135999828577042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,64,0.02804622385236952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,32,0.027680888772010803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,12288,0.3465013239118788
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,10240,0.34834665722317165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,10240,0.2860684394836426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,16384,0.4530791176689996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,16384,0.5553804503546821
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,12288,0.4850311279296875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,8192,0.283914672003852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,7168,0.2337902122073703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,8192,0.23388444052802193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,6144,0.26173777050442165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,5120,0.21468799644046357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,6144,0.16107466485765246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,5120,0.13450311289893255
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,7168,0.19052978356679282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,4096,0.1684417724609375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,3584,0.1459884378645155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,3584,0.10120444165335761
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,2560,0.08210933208465576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,3072,0.08989155292510986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,4096,0.1181866725285848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,3072,0.10929689142439102
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,1536,0.05115911033418444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,2560,0.0797662205166287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,1024,0.03585244549645318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,2048,0.06836533546447754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,768,0.01808177762561374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,2048,0.07403733332951863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,65536,1.6227991316053603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,768,0.03869955407248603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,1536,0.05869422356287638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,256,0.009072889056470659
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,128,0.006812444577614467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,512,0.03472000029351976
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,64,0.005594666633341048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,1024,0.042061332199308604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,512,0.012858666479587555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,32,0.0037431112594074676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,256,0.031034668286641438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,128,0.028240889310836792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,32,0.027087110612127516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,64,0.027013333307372198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,65536,2.1557155185275607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,12288,0.3544150988260905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,12288,0.24619110425313315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,16384,0.4471964306301541
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,10240,0.27861422962612575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,10240,0.19750666618347168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,16384,0.358550230662028
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,7168,0.19548889001210532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,8192,0.22618754704793295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,8192,0.15372089544932047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,6144,0.17098577817281088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,7168,0.1257315609190199
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,6144,0.10475999779171413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,5120,0.0885431104236179
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,5120,0.13255022631751165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,3584,0.09344266520606147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,4096,0.10771644115447998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,65536,1.117818620469835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,3584,0.06723555591371325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,4096,0.08031377527448866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,2560,0.0685244467523363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,3072,0.06059022082222832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,2048,0.05589689148796929
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,1536,0.04284444451332092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,3072,0.09026933378643459
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,2560,0.05398044321272108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,1024,0.02110133402877384
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,2048,0.04859377940495809
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,512,0.010851555400424533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,768,0.015126221709781222
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,256,0.007170666423108842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,1024,0.02823822365866767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,128,0.0052977779673205475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,1536,0.032631112469567194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,64,0.004348444441954295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,256,0.022167111436525982
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,512,0.024546666277779475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,32,0.003265777809752358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,768,0.026960889498392742
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,65536,1.7331795162624781
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,64,0.019909333851602342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,32,0.019559999306996662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,128,0.020251555575264823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,10240,0.2412275473276774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,12288,0.31284356117248535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,12288,0.21635822455088297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,16384,0.28643377621968585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,16384,0.41950755649142796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,8192,0.2001955509185791
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,8192,0.128330667813619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,7168,0.16917067103915742
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,10240,0.1720186604393853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,7168,0.10780977540545994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,6144,0.093849778175354
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,65536,1.0197440253363717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,6144,0.1570720010333591
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,5120,0.08028266827265422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,5120,0.13088177310095891
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,3072,0.07305333349439833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,4096,0.06763377454545763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,3584,0.061885330412122935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,3072,0.05902666515774197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,2560,0.060580445660485156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,2048,0.04965955681271023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,3584,0.09254311190711127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,1536,0.038112001286612615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,2560,0.04840711090299818
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,1024,0.017234666479958426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,4096,0.10809777842627631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,768,0.013908444179428948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,65536,1.5256498124864366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,512,0.012091555529170565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,2048,0.0391884446144104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,256,0.008597333398130205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,768,0.025535111625989277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,1536,0.03170311119821336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,512,0.02315555512905121
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,256,0.02151288919978672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,64,0.0038248888320393036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,1024,0.02697333362367418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,32,0.003206222214632564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,128,0.019303111566437613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,128,0.005516444643338521
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,64,0.018831999765502084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,32,0.01852799952030182
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,10240,0.20951910813649496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,10240,0.14146844546000162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,16384,0.33847644593980575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,16384,0.24645866288079155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,12288,0.19956444369422066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,12288,0.29696088367038304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,8192,0.17297066582573783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,7168,0.1438479953342014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,8192,0.1085688869158427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,5120,0.10187644428677028
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,7168,0.09217511283026801
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,4096,0.08554844723807441
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,6144,0.13889777660369873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,6144,0.0930595530403985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,3584,0.07189777824613783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,3072,0.0625857777065701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,4096,0.0606551104121738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,5120,0.07746399773491754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,65536,0.9059164259168836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,2560,0.05283911029497782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,3072,0.05399822195370992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,2048,0.04332177837689718
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,2560,0.041024890210893415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,1536,0.03273511264059279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,2048,0.03428711162673102
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,3584,0.05623110797670153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,1024,0.0169813334941864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,512,0.010615111225181155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,768,0.013768889009952545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,256,0.008075555165608725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,1024,0.026160000099076167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,128,0.005107555538415909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,512,0.021964444054497614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,64,0.003527111063400904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,65536,1.3169484668307834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,32,0.0028942223224374983
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,128,0.018654222289721172
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,64,0.01806400054030948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,1536,0.030003554291195337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,768,0.0243848893377516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,256,0.020091555184788175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,32,0.018950222267044913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,16384,0.29010044203864205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,16384,0.20977421601613364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,12288,0.24333156479729545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,10240,0.11169332928127712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,12288,0.15299555990431044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,8192,0.1429848935869005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,10240,0.20645333660973442
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,7168,0.12597066826290554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,6144,0.10456533564461602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,65536,0.7964240180121528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,8192,0.10062666734059651
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,4096,0.07513778077231513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,7168,0.08743733167648315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,65536,1.1018969217936199
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,6144,0.07774755689832899
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,3584,0.05800355805291069
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,5120,0.10220533609390259
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,4096,0.05363022287686666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,2560,0.04202488727039761
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,5120,0.06265866756439209
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,2048,0.034272889296213783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,1536,0.016061334146393668
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,3584,0.05161066518889534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,3072,0.04196622305446201
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,3072,0.05596444341871473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,1024,0.01178311142656538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,2560,0.03783911135461595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,768,0.00958577791849772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,512,0.007355555891990662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,1536,0.029449777470694646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,256,0.0052044445441828836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,128,0.0040835556056764396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,2048,0.03378666771782769
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,1024,0.025447110335032146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,512,0.021743110484547083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,64,0.0033573332346147964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,768,0.02402488887310028
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,256,0.019253333409627277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,128,0.01822933389080895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,64,0.01761777698993683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,32,0.0031591111587153543
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,32,0.017995556195576985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,16384,0.23756000730726454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,12288,0.1846391095055474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,10240,0.14315644900004068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,16384,0.1473555564880371
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,12288,0.10209866364796956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,8192,0.11124800311194526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,7168,0.09883289204703437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,8192,0.06996177964740329
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,10240,0.08533689048555161
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,6144,0.08550400204128689
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,7168,0.06276800235112508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,4096,0.058839109208848744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,65536,0.8868213229709201
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,65536,0.6538035604688857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,4096,0.03963377740648057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,5120,0.0528417792585161
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,5120,0.07084444496366712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,6144,0.05586666531032986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,3584,0.05199822121196323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,3072,0.04591288831498888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,3584,0.03019555409749349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,2560,0.03372622198528714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,2048,0.017078222499953378
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,1536,0.013693333499961428
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,1024,0.010084444450007545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,3072,0.02773244513405694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,768,0.008340444829728868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,2048,0.02296888828277588
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,256,0.004849777867396672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,512,0.006592000110281839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,1024,0.017845332622528076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,2560,0.025785777303907607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,768,0.017094223035706412
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,128,0.004013333469629288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,512,0.015501333607567681
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,64,0.003040888864133093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,32,0.0028640000770489373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,256,0.014319111903508505
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,1536,0.020710221595234342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,128,0.013226666384273104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,64,0.013383111192120446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,32,0.01277511070171992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,12288,0.09368977944056193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,16384,0.12792444229125977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,12288,0.16898400253719756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,16384,0.21111999617682564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,10240,0.13454666402604845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,10240,0.08031111293368869
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,8192,0.09878310892317031
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,7168,0.08963733249240452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,8192,0.06599733564588758
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,6144,0.07599822017881605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,65536,0.6089386410183376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,5120,0.06492266390058729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,6144,0.0521982212861379
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,7168,0.06252800093756782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,65536,0.7822408676147461
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,4096,0.05350755651791891
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,3584,0.04629688792758518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,5120,0.04686933424737719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,3584,0.029713776376512315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,3072,0.03949066665437486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,2560,0.027616888284683228
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,2048,0.016859556237856548
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,3072,0.027299554811583623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,1024,0.009943111075295342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,4096,0.03265688816706339
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,1536,0.013367999758985309
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,768,0.00832444429397583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,1536,0.019925332731670804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,1024,0.017616000440385606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,2048,0.022642667094866436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,512,0.006528000036875407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,768,0.016563556260532804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,2560,0.02537511123551263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,128,0.0038568890757030914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,64,0.00305600009030766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,32,0.0027626665929953256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,64,0.013186666700575085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,512,0.015506666567590503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,256,0.0140364451540841
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,32,0.012621333201726278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,128,0.013376889129479727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,12288,0.12986932860480413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,256,0.004790222065316306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,16384,0.181731555196974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,16384,0.11442489094204372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,12288,0.0857342216703627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,10240,0.1027546657456292
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,8192,0.08482844299740261
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,10240,0.07412711117002699
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,7168,0.07871466875076294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,8192,0.060917331112755664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,65536,0.6845333311292859
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,65536,0.562213314904107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,7168,0.05413688884841072
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,6144,0.048435555564032666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,6144,0.06855289141337077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,5120,0.03761333227157593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,5120,0.057499554422166615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,3584,0.040115555127461754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,3072,0.030412445465723675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,2560,0.02015199926164415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,4096,0.031721777386135526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,3584,0.029488888051774766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,2048,0.016724444097942777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,3072,0.0269822229941686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,4096,0.05233155687650045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,1536,0.013247110777431063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,1024,0.009831111464235518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,2560,0.024451555477248296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,512,0.006424888968467712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,2048,0.02241511146227519
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,768,0.008151999778217739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,1024,0.01735466718673706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,1536,0.019883554842736986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,256,0.004671111288997862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,128,0.003649777836269803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,512,0.014875556031862894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,768,0.016580444243219163
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,64,0.0028595555987623003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,32,0.0026142222599850762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,128,0.012803555362754397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,256,0.013975110318925647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,64,0.012488000094890594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,32,0.012287110918098025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,16384,0.14332621627383763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,10240,0.08692977825800578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,12288,0.10503911309772068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,16384,0.10208000077141656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,12288,0.07729511128531562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,8192,0.07134933604134454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,7168,0.06272000074386597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,10240,0.06756177875730726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,6144,0.0550746652815077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,8192,0.05634488662083944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,7168,0.04943022131919861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,5120,0.04701333244641622
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,3584,0.03490399983194139
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,4096,0.03906133439805772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,6144,0.04165510998831855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,5120,0.035050666994518705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,65536,0.5823662016126845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,2560,0.017024889588356018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,3072,0.018555555078718398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,2048,0.014717333846622042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,3584,0.028155555327733357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,1536,0.011092444260915121
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,65536,0.5255946583218044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,4096,0.030425777037938435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,1024,0.008223999705579545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,2560,0.023687111006842718
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,3072,0.025968889395395916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,768,0.006884444504976273
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,1024,0.016586666305859882
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,512,0.005570666657553778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,256,0.0047733332547876565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,768,0.015255111787054272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,128,0.003391999958289994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,1536,0.01869155632125007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,512,0.014155555102560254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,64,0.0027262222849660446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,2048,0.02122666603989071
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,128,0.01183111137813992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,32,0.002575111058023241
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,256,0.013168888787428537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,64,0.011706666813956367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,32,0.0116462219092581
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,16384,0.10778222481409709
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,10240,0.07042488786909315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,16384,0.0929644438955519
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,12288,0.0830142233106825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,12288,0.07040266858206855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,10240,0.06234844525655111
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,8192,0.05776533153322008
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,7168,0.05125155713823107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,65536,0.47794310251871747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,8192,0.04907822277810839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,6144,0.04510755671395195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,7168,0.04352622230847677
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,5120,0.03867022196451823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,65536,0.49693955315483945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,6144,0.03906311260329352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,4096,0.02144266664981842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,3584,0.017672888106769986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,3072,0.016857778032620747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,2560,0.015368888775507608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,2048,0.013613333304723104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,2560,0.023277334041065637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,1536,0.009413333402739631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,5120,0.034635557068718806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,4096,0.030113776524861652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,1024,0.007060444189442529
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,3584,0.027826666831970215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,768,0.006020444548792309
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,3072,0.025518221987618342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,512,0.004937777916590373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,1536,0.018353778454992507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,1024,0.01589777734544542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,256,0.004562666846646203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,768,0.014933332800865173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,128,0.003173333282272021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,512,0.013691555294725629
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,64,0.0025475554996066624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,32,0.0023431111541059283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,2048,0.020861332615216572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,64,0.011174221833546957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,128,0.011345778074529437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,32,0.011191111471917896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,256,0.012512889173295764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,16384,0.09230577945709229
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,12288,0.06387377447552152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,16384,0.08665333191553752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,10240,0.05450133482615153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,12288,0.06630755795372857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,8192,0.045012444257736206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,7168,0.03761777612898085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,65536,0.3630239963531494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,6144,0.03284800052642822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,10240,0.06110044320424398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,8192,0.047790222697787814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,7168,0.04319377740224203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,4096,0.015612444943851896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,6144,0.03880000114440918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,65536,0.4627608723110623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,3584,0.014324444863531323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,4096,0.029767112599478826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,5120,0.01828444500764211
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,2560,0.011949333051840464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,3584,0.02733777794573042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,5120,0.03406577640109592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,2048,0.01586933268441094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,3072,0.01311288856797748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,1536,0.013071999781661563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,1024,0.009536888864305284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,2560,0.02292177743381924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,768,0.007864889171388414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,2048,0.020395555429988436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,1536,0.017874666386180453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,512,0.006011555592219035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,3072,0.025088000628683303
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,1024,0.015653333730167814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,768,0.014668444792429606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,128,0.0033235556135574975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,256,0.004388444539573458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,64,0.00254133323000537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,512,0.013264888690577613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,32,0.0022515555222829184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,256,0.01202311118443807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,128,0.011152889165613385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,32,0.010782221953074137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,64,0.011010666688283285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,16384,0.057373331652747266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,12288,0.044665777020984225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,16384,0.08414577775531346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,65536,0.2533831066555447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,8192,0.019664888580640156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,12288,0.06591288910971747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,10240,0.03966844413015578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,10240,0.0569102234310574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,7168,0.017138666576809354
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,6144,0.015784000356992085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,8192,0.04765244325002035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,5120,0.013426666458447775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,65536,0.4193289015028212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,4096,0.011666666302416058
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,6144,0.03877955675125122
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,3584,0.011030221978823343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,4096,0.029559112257427637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,3072,0.010205333431561789
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,2560,0.009285333255926767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,5120,0.03413777881198459
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,2048,0.00963199966483646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,3584,0.027436445156733196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,7168,0.04311466548177931
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,1536,0.007991111112965478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,1024,0.006206222292449739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,768,0.00535999983549118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,2560,0.022783999641736347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,2048,0.020303999384244282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,512,0.004400889078776042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,1536,0.018009778526094224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,256,0.0035333335399627686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,1024,0.015432000160217285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,3072,0.025115556187099878
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,768,0.014481777946154276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,512,0.013224889006879596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,128,0.002834666727317704
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,64,0.0022355556074115965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,32,0.0020853332761261198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,256,0.011937778029176923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,64,0.010835555692513784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,16384,0.04454400142033895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,32,0.01072622256146537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,128,0.011065777805116443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,10240,0.018055111169815063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,12288,0.033018665181265935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,16384,0.0840195549858941
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,65536,0.1984835598203871
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,8192,0.015020444989204407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,12288,0.06586222516165839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,10240,0.056946668359968394
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,7168,0.013924444715181986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,6144,0.012612444659074148
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,5120,0.011330666641394297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,4096,0.010784000158309937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,8192,0.04818755719396803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,7168,0.04320089022318522
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,3584,0.00998755544424057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,6144,0.038711110750834145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,5120,0.03410311208830939
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,4096,0.029534220695495605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,3584,0.027600000301996868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,2560,0.00832266691658232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,65536,0.40449955728318954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,2048,0.011504000259770287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,3072,0.024960888756646052
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,1536,0.009201777478059133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,3072,0.0090844440791342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,2560,0.02271822260485755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,1024,0.006798222247097228
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,768,0.005658666706747479
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,512,0.004387555436955558
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,2048,0.02050488856103685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,256,0.0032835555159383347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,1536,0.017829333742459614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,1024,0.015463999576038785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,768,0.014527999692493014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,128,0.0025928889711697898
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,64,0.0021359999146726397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,32,0.002346666736735238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,512,0.013186666700575085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,128,0.011129777464601727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,256,0.011930666863918304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,16384,0.02723199956946903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,64,0.010846222440401712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,32,0.010757333702511258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,12288,0.015019555886586508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,10240,0.013378666506873237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,8192,0.011391110718250275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,65536,0.11391999986436631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,16384,0.08401599857542251
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,7168,0.010979555547237396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,10240,0.057251552740732826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,12288,0.0657786660724216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,6144,0.010957332948843638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,8192,0.04751111070315043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,5120,0.009546666509575313
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,4096,0.008392888638708327
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,7168,0.0430391099717882
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,6144,0.03852444556024339
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,3584,0.007975111405054728
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,5120,0.03410222133000692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,3072,0.007442666424645319
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,2560,0.006801777829726537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,4096,0.029636445972654555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,2048,0.006267555471923616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,65536,0.39571465386284727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,3584,0.027228444814682007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,3072,0.024921778175565932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,1536,0.008958222137557136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,2560,0.022646221849653456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,1024,0.006616888774765863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,768,0.005428444594144821
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,512,0.004242666479614046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,2048,0.020290666156344943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,256,0.003083555648724238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,1024,0.015323556131786771
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,768,0.014392889208263822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,128,0.002509333400262727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,64,0.002104000084929996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,1536,0.017712000343534682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,512,0.013215999636385175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,32,0.0023164444913466773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,256,0.011906666888131036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,128,0.011071999867757162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,64,0.010827555424637265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,65536,0.054767111937204994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,32,0.010686222049925061
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,12288,0.010424888796276517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,16384,0.013187555803192986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,10240,0.009553777674833933
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,8192,0.00850222259759903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,16384,0.0840266678068373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,7168,0.007937777373525832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,10240,0.05692978037728203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,12288,0.06636089086532593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,6144,0.0074346669846110875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,8192,0.04773422082265218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,7168,0.04300711221165127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,5120,0.006776000062624614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,4096,0.006176888942718506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,3584,0.005904888941182031
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,3072,0.005646222167544895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,6144,0.03858133488231235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,2560,0.005395555661784277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,5120,0.034316443734698825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,4096,0.02963199880388048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,65536,0.35512712266710067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,3584,0.027217777238951787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,2048,0.004984000076850255
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,3072,0.024889777104059856
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,1536,0.005103999955786599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,1024,0.006433777925040986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,768,0.005223111146026188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,2560,0.022648000054889258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,2048,0.020264888803164165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,512,0.004087111188305749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,256,0.0029128889242808023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,128,0.0023431111541059283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,1536,0.017841777867741056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,1024,0.015308444698651632
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,768,0.014446222119861178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,512,0.013006221916940479
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,64,0.0022453332526816265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,32,0.002181333386235767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,256,0.011897777517636618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,65536,0.03751111030578613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,16384,0.009200000100665623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,128,0.011032889286677042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,64,0.010965333216720156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,32,0.010795555180973478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,12288,0.010759111079904767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,10240,0.009783999787436591
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,8192,0.008472888833946651
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,7168,0.00832266691658232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,16384,0.08415377802318996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,6144,0.007709333466158972
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,12288,0.06582578023274739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,10240,0.05707999732759264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,8192,0.04748355680041843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,5120,0.007171555525726742
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,4096,0.006656000183688269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,7168,0.04320089022318522
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,3584,0.006027555714050929
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,6144,0.03880177934964498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,3072,0.0058622223635514574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,5120,0.03407199846373664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,65536,0.34743555386861164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,4096,0.02940355406867133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,2560,0.005418666534953647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,2048,0.005239999956554837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,3072,0.025031111306614343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,1536,0.004483555754025777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,1024,0.005611555443869696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,2560,0.022597332795461018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,2048,0.02015111181471083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,768,0.004691555682155821
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,1536,0.0177520000272327
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,512,0.0037208890749348533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,3584,0.027339556150966223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,1024,0.01540177729394701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,256,0.0027786667148272195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,768,0.01426666643884447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,128,0.0023022221608294383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,64,0.0021822222818930945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,512,0.013015111287434896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,32,0.0021377777059872947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,256,0.01181422256761127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,65536,0.02040177749262916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,128,0.011058666639857821
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,64,0.010904889139864178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,16384,0.008784000244405534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,32,0.010760889285140567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,12288,0.007637333538797166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,10240,0.007287999822033777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,8192,0.006628444625271692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,7168,0.006223111102978389
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,10240,0.056663109196556934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,6144,0.006086222413513396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,12288,0.06553600231806438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,16384,0.08383555544747247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,8192,0.04728088776270548
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,5120,0.005585777676767773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,7168,0.04288799895180596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,4096,0.005256888767083486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,3584,0.004987555659479565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,65536,0.34657777680291074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,3072,0.0046471113132105935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,6144,0.03838222225507101
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,5120,0.033879998657438494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,4096,0.029172443681293066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,3584,0.027104889353116352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,2560,0.0047893333766195504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,2048,0.004545777622196409
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,1536,0.003944000022278892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,1024,0.003642666671011183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,3072,0.02476355599032508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,2560,0.022463111413849726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,768,0.004558222161398994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,2048,0.01995022263791826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,512,0.0036311112344264984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,256,0.0026951111439201566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,1536,0.017634666628307767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,1024,0.015063110325071545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,128,0.002242666565709644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,768,0.014044443766276041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,64,0.002165333264403873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,512,0.01297511077589459
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,32,0.0021422221842739317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,256,0.011692444483439127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,65536,0.01646844380431705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,128,0.01088177743885252
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,64,0.010728889041476779
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,16384,0.007024889190991719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,32,0.010701333483060202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,10240,0.00611111107799742
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,12288,0.00602311102880372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,8192,0.005531555662552516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,7168,0.005038222091065513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,16384,0.08443111181259155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,10240,0.057012445396847196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,12288,0.06560355424880981
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,6144,0.0048177776237328844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,8192,0.04749777913093567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,7168,0.04281599985228645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,5120,0.00498488876554701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,4096,0.00481244424978892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,3584,0.004344888859324985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,3072,0.004176889028814104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,65536,0.36231467458936906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,6144,0.03848177856869168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,5120,0.03378222386042277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,2560,0.004059555629889171
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,4096,0.02915466825167338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,3584,0.02701955537001292
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,2048,0.0038577777643998465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,1536,0.004032888760169347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,1024,0.0035128887328836652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,3072,0.02459200057718489
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,768,0.004496888981925117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,2560,0.02233244478702545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,512,0.003580444388919406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,2048,0.019839111301634047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,1536,0.01742488808102078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,256,0.002809777855873108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,128,0.0022151112142536375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,1024,0.015096000499195524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,768,0.014109333356221518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,512,0.012906666431162091
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,64,0.0021333334346612296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,32,0.0020951111283567217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,256,0.011565333439244164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,128,0.010856889188289642
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,32,0.010781333678298526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,64,0.010670222342014313
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,10240,1.311489741007487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,12288,1.5958115259806316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,16384,2.077852461073134
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,10240,2.061409844292535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,8192,1.6689057879977758
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,8192,0.9983911514282227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,12288,2.725224812825521
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,7168,0.8315724266899956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,6144,0.6999564700656467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,7168,1.4577947192721898
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,16384,3.2657563951280384
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,5120,0.5754817856682671
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,6144,1.293365372551812
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,4096,0.8542213439941406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,4096,0.4987813101874457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,5120,1.1493386162651908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,3584,0.4390862252977159
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,3072,0.386791123284234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,2560,0.5509991115993923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,3072,0.722541332244873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,3584,0.8274728986952039
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,2048,0.2638888888888889
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,2560,0.3348124557071262
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,2048,0.48767911063300234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,1024,0.2401662137773302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,1536,0.2114844454659356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,256,0.05288799934917026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,1024,0.16030311584472656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,768,0.20150933000776503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,768,0.12503822644551596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,1536,0.37198665406968856
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,256,0.10041599803500706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,512,0.11780977249145508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,512,0.10608088970184326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,64,0.012844444149070315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,32,0.014288889037238227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,128,0.10368444522221883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,32,0.09655111365848118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,128,0.03458222084575229
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,64,0.10133600234985352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,16384,0.5772257910834419
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,12288,0.42502933078342015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,12288,0.7922995355394152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,10240,0.37835110558403867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,16384,0.8603279855516223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,10240,0.601079093085395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,8192,0.29204000367058647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,6144,0.33134934637281627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,7168,0.2527013354831272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,7168,0.3830222288767497
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,6144,0.2379697693718804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,8192,0.48595910602145725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,5120,0.26983287599351674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,4096,0.1343244446648492
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,4096,0.21893955601586235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,3072,0.16247467199961343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,3584,0.1149653328789605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,5120,0.19105333752102324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,2560,0.1312275595135159
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,3584,0.21047643820444742
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,2048,0.10534399747848511
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,2560,0.08460088570912679
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,1536,0.08029955625534058
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,65536,1.8666125403510199
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,1024,0.0551146666208903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,2048,0.07690311140484281
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,768,0.04182933436499702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,3072,0.10780355665418838
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,1536,0.062408890989091664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,128,0.0061039999127388
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,768,0.02824977702564663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,64,0.004651555584536659
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,256,0.022824888428052265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,1024,0.04706577791108025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,512,0.024803555674023096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,512,0.022201778160201177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,128,0.02088800072669983
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,32,0.004513777792453766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,256,0.008396444221337637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,32,0.019660444723235238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,64,0.019918221566412184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,12288,0.3304142157236735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,16384,0.417996433046129
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,12288,0.533871120876736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,10240,0.4483991199069553
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,16384,0.6754826439751519
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,10240,0.2723120053609212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,65536,3.2968898349338107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,8192,0.21571821636623809
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,8192,0.3307617770300971
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,7168,0.18459288279215494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,7168,0.31515643331739635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,6144,0.2562862237294515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,5120,0.12161955568525527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,3584,0.1407359970940484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,4096,0.1659031046761407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,5120,0.22799200481838652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,6144,0.16489511066012913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,65536,1.4777644475301106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,4096,0.10742044448852539
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,2560,0.10063199864493476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,2048,0.08192621999316745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,3584,0.0970311098628574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,2560,0.07357422510782878
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,3072,0.1342008908589681
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,2048,0.056827558411492236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,3072,0.08334755235248142
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,768,0.02576622201336755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,1536,0.047825776868396334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,1024,0.02612000041537815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,1024,0.047487109899520874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,256,0.007289778027269576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,1536,0.06897066699133979
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,512,0.010840000377760993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,768,0.024149333437283833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,128,0.005223999834722943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,64,0.004385777645640903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,512,0.02281422250800663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,128,0.018545778261290658
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,32,0.003920888735188378
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,256,0.02037333283159468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,32,0.018405333161354065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,64,0.017827555537223816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,65536,2.4884791904025607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,12288,0.2761582268608941
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,10240,0.3463911215464274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,16384,0.5483893288506402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,12288,0.439807997809516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,16384,0.3855360084109836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,10240,0.2249608834584554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,8192,0.2837582164340549
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,8192,0.17558844884236655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,7168,0.15600177976820204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,7168,0.3164159986707899
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,6144,0.26234934065077037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,5120,0.11012799872292413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,5120,0.2127493355009291
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,6144,0.1301386621263292
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,4096,0.1408702267540826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,4096,0.0834275550312466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,65536,1.2585378222995334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,3584,0.13134311305152044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,3584,0.07455822494294909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,3072,0.11521244049072266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,2560,0.08886577685674031
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,3072,0.06641511122385661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,1536,0.058752887778811984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,2048,0.04938844508594937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,2048,0.07765422264734904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,2560,0.06299822198020087
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,1024,0.02540622154871623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,768,0.018387556076049805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,1024,0.039990223116344877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,768,0.023740443918440077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,128,0.005205333232879639
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,1536,0.03402933478355408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,256,0.007103111180994246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,512,0.010672888822025724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,64,0.004492444296677907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,256,0.019311999281247456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,65536,2.0789101918538413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,512,0.021835555632909138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,32,0.0032915555768542816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,128,0.017995556195576985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,64,0.017607111069891188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,32,0.0177511109246148
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,12288,0.21670577261182997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,10240,0.2694257895151774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,10240,0.17497866683536104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,16384,0.42853334214952254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,12288,0.3888266616397434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,16384,0.3057253360748291
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,8192,0.22194577587975395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,7168,0.10727110836240981
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,7168,0.19204889403449166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,5120,0.1344053347905477
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,6144,0.09320977661344741
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,6144,0.1821111175749037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,8192,0.1372142235438029
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,4096,0.10389333301120335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,3584,0.09115822447670831
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,65536,1.039804458618164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,3072,0.08824621968799168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,3072,0.05337511168585884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,3584,0.06495289007822673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,2560,0.06604088677300347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,4096,0.06642133659786649
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,1024,0.01664266652531094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,1536,0.04569688770506117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,2048,0.060007108582390674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,5120,0.08739022413889568
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,768,0.010039110978444418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,512,0.007815111014578078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,1024,0.019701333509551156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,2048,0.03723466727468703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,256,0.00563733321097162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,2560,0.049583110544416636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,128,0.00434666665063964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,512,0.016971555021074083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,768,0.01863911085658603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,1536,0.02236000034544203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,32,0.0029564443975687027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,256,0.015446222490734525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,65536,1.6735137303670247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,64,0.0036524443162812125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,64,0.014090667168299357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,32,0.014044443766276041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,128,0.014462222655614218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,12288,0.31318667199876576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,12288,0.18657510810428193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,16384,0.2553653452131483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,10240,0.2609475453694661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,16384,0.41780532730950254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,8192,0.19448088275061715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,10240,0.14273422294192845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,8192,0.10637955533133613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,6144,0.14158222410413954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,7168,0.18027110894521078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,6144,0.08318044741948445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,7168,0.10429510805341934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,5120,0.11345422267913818
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,65536,0.9328720304701063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,5120,0.07064088847902086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,4096,0.059691554970211454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,3584,0.0891537798775567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,4096,0.10305600033866034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,3072,0.06867377625571357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,3584,0.052975111537509494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,2560,0.05779733260472616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,2560,0.04365333252482944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,1536,0.021643555826610986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,1024,0.011679111255539788
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,65536,1.4712631437513564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,2048,0.02700622214211358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,3072,0.050888889365726046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,768,0.009686222506894005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,2048,0.05233600073390537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,1024,0.018963555494944256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,256,0.005321777943107817
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,128,0.004266666869322459
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,512,0.007937777373525832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,1536,0.04016444418165419
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,512,0.015969778100649517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,64,0.0035377778112888336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,256,0.01499288943078783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,32,0.0032960000551409195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,128,0.01370133293999566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,64,0.01348622226052814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,768,0.017877333694034152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,32,0.013434666726324292
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,16384,0.2128666639328003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,12288,0.27249333593580455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,10240,0.21239644951290557
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,12288,0.1612915595372518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,16384,0.33894933594597715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,10240,0.11407732963562012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,8192,0.16543022791544595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,8192,0.09072355429331462
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,7168,0.15623377429114446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,6144,0.12136977248721653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,5120,0.09989421897464329
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,6144,0.0707119968202379
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,4096,0.07938133345709907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,7168,0.08736089203092788
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,4096,0.05111644334263272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,3072,0.06010578076044718
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,5120,0.06658755408393012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,65536,0.8213129043579102
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,2560,0.04994044370121426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,3072,0.04128177629576789
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,2048,0.04546311166551378
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,3584,0.07720444599787395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,2560,0.0303146673573388
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,1536,0.02904799911710951
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,3584,0.0461733341217041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,768,0.008987555901209513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,1024,0.010776888993051318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,512,0.0070222218831380205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,65536,1.2741271124945748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,1024,0.017815111411942374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,256,0.005197333378924264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,1536,0.020032889313167997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,768,0.016958221793174744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,512,0.015784000356992085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,64,0.003265777809752358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,256,0.01426755554146237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,32,0.003105777833196852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,2048,0.022666666242811415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,32,0.013091555900043912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,128,0.004093333250946469
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,64,0.012970666918489667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,128,0.013063111239009433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,16384,0.2725004355112712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,12288,0.2162319951587253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,10240,0.17157244682312012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,12288,0.1286897791756524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,10240,0.09760000308354695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,16384,0.17928711573282877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,7168,0.13785333103603786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,6144,0.1020159986284044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,8192,0.08161244127485487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,8192,0.13910755846235487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,7168,0.07746311028798421
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,5120,0.08611110846201579
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,4096,0.07093599769804212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,6144,0.06299733453326754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,5120,0.05558933152092827
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,65536,0.7026693556043836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,3072,0.04982666505707634
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,2560,0.04169244567553202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,3584,0.04160533348719279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,3072,0.03049244483311971
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,3584,0.06938755512237549
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,65536,1.0534462398952908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,1536,0.013697778185208639
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,2048,0.03399466805987888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,4096,0.04970399869812859
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,2560,0.02457422183619605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,1024,0.01017955525053872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,768,0.008563555777072906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,512,0.006704888823959563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,1536,0.01956533392270406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,1024,0.01776444415251414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,2048,0.02201244400607215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,512,0.015244444211324057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,128,0.0038853333228164246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,768,0.016743999388482835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,256,0.005065777649482091
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,32,0.002842666581273079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,64,0.003168888803985384
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,256,0.0140346669488483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,128,0.013281777501106262
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,32,0.013087111214796701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,12288,0.1799164480633206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,64,0.012882666455374824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,16384,0.2275031142764621
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,16384,0.13337955209943983
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,12288,0.10001689195632935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,10240,0.13215643829769558
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,8192,0.10624621974097358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,7168,0.09300355778800116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,10240,0.08989866574605306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,8192,0.06938933663898043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,7168,0.06193066967858208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,6144,0.05532177620463901
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,5120,0.07788444227642484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,4096,0.05629511011971367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,65536,0.6028781996832954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,5120,0.04808800088034736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,6144,0.08227822515699598
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,2560,0.03245866629812453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,4096,0.03853066762288412
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,65536,0.8485431141323514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,3584,0.028522666957643297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,3072,0.044065776798460216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,2048,0.014998222390810648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,3072,0.0258577780591117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,1536,0.012323555847009024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,1024,0.008971555365456475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,3584,0.05526399943563673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,2560,0.02342133389578925
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,2048,0.020979555116759405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,768,0.0075137780772315125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,256,0.004561777744028303
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,512,0.005968889014588461
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,1024,0.016819554898473952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,768,0.015523556205961438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,512,0.014190221826235453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,256,0.013001778059535556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,1536,0.018699554933442008
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,64,0.0029760001020299066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,32,0.0028071111689011254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,128,0.003705777641799715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,64,0.01184088902340995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,128,0.012138666378127204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,12288,0.1517475578520033
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,16384,0.1905599964989556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,12288,0.0914906660715739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,16384,0.12713689274258083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,32,0.01200977795653873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,10240,0.12794578075408936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,8192,0.10601777500576443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,10240,0.07747822337680392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,6144,0.0735608869128757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,7168,0.05710311068428887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,65536,0.5586969057718912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,5120,0.06272533204820421
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,7168,0.09416888819800483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,8192,0.06403644217385186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,6144,0.05108977688683403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,4096,0.029606223106384277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,5120,0.04501333170466953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,3584,0.04428355561362373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,65536,0.7603813277350532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,2560,0.027564444475703772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,3072,0.033717334270477295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,2048,0.013648000028398303
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,3072,0.025236444340811834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,3584,0.027466666367318895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,1536,0.011352889239788055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,2048,0.02054222259256575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,2560,0.022984888818528917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,768,0.007071110937330458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,4096,0.05539733171463013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,1536,0.018194667167133756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,512,0.00572622236278322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,1024,0.01608533329433865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,256,0.004488888714048597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,1024,0.008416888614495596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,128,0.003524444583389494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,512,0.01386666629049513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,64,0.0027955555253558685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,32,0.002571555475393931
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,128,0.011659555964999728
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,256,0.012667555775907306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,64,0.011648889217111798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,768,0.015003555350833468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,32,0.011376889215575324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,12288,0.12147200107574463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,16384,0.16527556048499212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,10240,0.10226577520370483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,16384,0.10749510924021403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,12288,0.08965244558122422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,8192,0.08349866337246364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,7168,0.07339821921454535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,8192,0.059191114372677274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,10240,0.07122666968239678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,6144,0.0641626649432712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,65536,0.5122053358289931
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,7168,0.052419556511773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,65536,0.6490951114230686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,4096,0.043508443567487926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,5120,0.05490933193100823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,6144,0.04692088895373874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,5120,0.03584444522857666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,3072,0.02896889050801595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,4096,0.02942755487230089
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,2048,0.012513777448071374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,2560,0.01577422188387977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,3584,0.03865955604447259
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,3072,0.02499288817246755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,1024,0.007639110916190677
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,2560,0.02269777821169959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,1536,0.010178666975763109
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,3584,0.027302222119437322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,1536,0.017952000101407368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,768,0.006478222293986215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,512,0.005391110976537068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,1024,0.01590044465329912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,2048,0.020463999774720933
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,128,0.0033084443873829315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,256,0.004675555560323927
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,768,0.014783999986118741
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,64,0.002761777697337998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,32,0.002464888824356927
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,256,0.012299555871221753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,512,0.013728888498412238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,128,0.011408888631396823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,64,0.011195555329322815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,32,0.011247110863526663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,12288,0.09941866662767197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,16384,0.13341600365108913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,12288,0.0747928884294298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,10240,0.08435910940170288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,16384,0.09815200169881184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,10240,0.06552088922924466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,8192,0.06843199994828966
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,7168,0.060402664873335093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,8192,0.05419910947481791
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,6144,0.052427556779649526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,7168,0.047797332207361855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,4096,0.0370399985048506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,5120,0.04500177833769056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,65536,0.5452088779873318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,6144,0.03794577717781067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,65536,0.48293330934312606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,5120,0.033435556623670794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,3584,0.0328693323665195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,3072,0.016229333149062265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,2560,0.013462222284740873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,2048,0.01092088884777493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,4096,0.02923822071817186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,3584,0.027012444204754297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,2560,0.022509333160188463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,1024,0.007153777612580194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,768,0.006035555568006303
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,1536,0.009285333255926767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,1536,0.017902221944597032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,768,0.014713777436150445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,256,0.004089777668317159
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,2048,0.02033422225051456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,3072,0.024731554918819006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,512,0.013416888813177744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,1024,0.015768888923856948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,128,0.003168888803985384
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,512,0.005122666557629903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,64,0.0026168889469570587
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,32,0.002487999904486868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,256,0.012285333540704517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,128,0.011221333510345884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,64,0.010964444114102257
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,32,0.010985777609878115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,16384,0.1047288907898797
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,12288,0.08071821928024292
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,10240,0.06765066915088229
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,16384,0.09044355816311306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,12288,0.06796533531612821
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,8192,0.055332445436053805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,10240,0.05905155340830485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,7168,0.048968000544442065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,6144,0.04273688793182373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,8192,0.049117333359188504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,7168,0.04179288943608602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,5120,0.03653511073854234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,65536,0.4563591215345595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,4096,0.018026666508780587
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,65536,0.44093865818447536
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,6144,0.03755822115474277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,5120,0.03334133492575751
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,3584,0.014091556270917257
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,2560,0.02167822255028619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,4096,0.028838220569822524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,2048,0.016171556380059984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,3584,0.026733333865801494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,3072,0.012524444195959302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,1536,0.013753777576817406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,2048,0.019881778293185763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,3072,0.024569777978791132
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,768,0.008231999973456064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,1024,0.009764444496896531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,2560,0.022295110755496558
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,1024,0.015356444650226168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,1536,0.017705778280893963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,512,0.004606222112973531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,256,0.004456888884305954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,128,0.0033119999700122406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,768,0.014330666926172046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,64,0.0026506665680143568
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,32,0.0023146667000320223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,512,0.013088000317414602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,256,0.01202044470442666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,128,0.01111644423670239
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,64,0.010837333069907295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,32,0.010751999914646149
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,16384,0.08064533604515924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,12288,0.062149332629309766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,10240,0.052278220653533936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,16384,0.08242311080296834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,12288,0.06425599919425116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,8192,0.04305244485537211
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,7168,0.03797511259714762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,65536,0.3412862353854709
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,8192,0.046577778127458364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,7168,0.04182044333881802
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,5120,0.014030221435758801
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,6144,0.03330755564901564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,4096,0.012448888685968188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,10240,0.05487199955516391
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,6144,0.0376160012351142
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,3584,0.01146399974822998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,5120,0.03317777646912469
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,65536,0.41910754309760195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,3072,0.010617777705192566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,2560,0.01166311071978675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,3584,0.026644445127911035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,2048,0.009864889085292816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,1536,0.008200888832410177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,1024,0.0063475556671619415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,4096,0.028730667299694482
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,3072,0.024523556232452393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,2048,0.01982311076588101
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,768,0.005592000153329637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,2560,0.02216266592343648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,1536,0.017475555340449016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,512,0.004680888934267892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,128,0.00305600009030766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,768,0.014242667290899487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,1024,0.015409777561823526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,256,0.011915555430783166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,64,0.0023742222951518162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,32,0.002285333350300789
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,128,0.01109955542617374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,512,0.01312711089849472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,256,0.0036657779581016968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,64,0.010795555180973478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,32,0.010749333434634738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,12288,0.042484445704354175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,16384,0.05476444297366672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,10240,0.0362062222427792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,7168,0.015281778242852954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,16384,0.08073955774307251
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,12288,0.06367022461361356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,8192,0.01942222151491377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,65536,0.24284622404310438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,10240,0.05474311113357544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,5120,0.012155555188655853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,6144,0.013713777893119387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,8192,0.046132445335388184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,4096,0.009939555492666032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,7168,0.04190133346451653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,6144,0.03747555613517761
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,5120,0.033152000771628484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,3072,0.008616888688670265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,4096,0.028928889168633357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,3584,0.009366222553782994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,3584,0.026712000370025635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,2560,0.014114666316244336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,65536,0.3697964350382487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,1536,0.009307555854320526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,2048,0.011560888753996955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,3072,0.024316444993019104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,768,0.005704888867007361
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,2560,0.022172444396548804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,2048,0.019725332657496136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,512,0.004467555632193883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,1024,0.006808888994985157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,256,0.0032720000793536506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,1024,0.01536622146765391
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,768,0.014161777165200977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,1536,0.017470222380426195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,128,0.0026026666164398193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,64,0.0021911110315057966
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,512,0.013081777426931592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,32,0.0020711111525694528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,256,0.011921777493423887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,128,0.010987555815113915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,32,0.010791999598344168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,16384,0.0429440008269416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,64,0.010809777511490716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,12288,0.032893333170149065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,10240,0.014527110589875115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,65536,0.1830693350897895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,8192,0.013648000028398303
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,7168,0.012642666697502136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,16384,0.08099289072884454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,10240,0.05471822288301256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,12288,0.06366578075620863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,6144,0.011527111132939657
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,8192,0.04595466785960727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,5120,0.01204622205760744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,4096,0.010384889112578498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,3584,0.009296889106432596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,7168,0.04175644450717502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,6144,0.03754666778776381
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,65536,0.3507395585378011
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,5120,0.03317600157525804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,3072,0.008986666798591614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,4096,0.02862400147649977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,2560,0.00865955568022198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,3584,0.026572444372706946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,2048,0.011409777734014722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,1536,0.009073778159088558
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,1024,0.006678222368160884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,768,0.005550222264395819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,2560,0.02202844454182519
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,2048,0.01960000064637926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,3072,0.024329778220918443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,1536,0.017482666505707633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,512,0.004314666820896996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,256,0.0031644445326593188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,128,0.002554666664865282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,64,0.0020826665891541377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,768,0.014123555686738757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,32,0.0022719999154408774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,1024,0.015259555644459195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,256,0.011726222104496427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,128,0.011050666371981302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,512,0.01295733369059033
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,64,0.01092266705301073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,32,0.01070844464831882
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,12288,0.013563555147912769
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,10240,0.012044444680213928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,65536,0.1071395542886522
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,8192,0.010475555227862464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,16384,0.020941333638297185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,16384,0.08064089218775432
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,7168,0.00962755580743154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,10240,0.05468089050716824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,6144,0.008732444710201686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,12288,0.06375288963317871
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,5120,0.007876444194051955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,7168,0.0417795545525021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,4096,0.007010666860474481
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,8192,0.04598311252064175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,6144,0.03751911057366265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,5120,0.033053331904941134
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,3584,0.006694222076071634
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,3072,0.006342222293217977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,65536,0.3310408857133654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,2560,0.006002666635645761
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,4096,0.028575999869240656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,2048,0.005446222093370225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,3584,0.02665422194533878
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,1536,0.008901333643330468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,3072,0.02436977790461646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,2560,0.022036444809701707
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,1024,0.006504888749784893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,768,0.005373333477311664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,512,0.004156444635656145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,256,0.003084444544381566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,1536,0.017337777548366122
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,2048,0.019741333193249173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,1024,0.015257777439223396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,768,0.014157333307796054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,128,0.002373333399494489
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,512,0.0129457778400845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,64,0.002016000035736296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,32,0.0021955555097924336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,256,0.011735999749766456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,65536,0.05267911156018575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,32,0.010644444160991244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,64,0.010799999866220685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,16384,0.010445333189434476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,128,0.01110311100880305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,10240,0.008104888929261101
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,8192,0.007250666618347168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,12288,0.008943110704421997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,7168,0.010229333407349056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,12288,0.06361155377493964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,10240,0.0547653337319692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,6144,0.006502222269773483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,8192,0.04621511035495334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,16384,0.08081689145829943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,5120,0.008565333154466417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,7168,0.042457779248555504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,4096,0.00546577779783143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,65536,0.30967643525865346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,3584,0.005681777579916849
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,3072,0.006996444529957241
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,6144,0.03743288914362589
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,2560,0.0063999998900625445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,5120,0.0332293344868554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,4096,0.028726223442289565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,2048,0.005751111027267244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,3584,0.026847110854254827
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,1024,0.0057813334796163775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,1536,0.0047848886913723415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,3072,0.024275556206703186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,2560,0.022054221895005967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,2048,0.019578667150603402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,768,0.004835555536879433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,512,0.0038328886859946777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,1536,0.017468444175190397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,256,0.0029146667155954572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,128,0.0023493334237072202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,768,0.014192889134089151
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,1024,0.015152888165579902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,512,0.012905778156386482
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,64,0.0021422221842739317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,256,0.011701333026091257
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,32,0.0021502222451898786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,128,0.011085333095656501
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,65536,0.032960888412263654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,64,0.010816888676749336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,16384,0.010878221856223213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,12288,0.009275555610656738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,32,0.01070666644308302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,10240,0.008557333714432186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,8192,0.007449777589903937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,7168,0.007207999626795451
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,6144,0.006671111202902264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,12288,0.06362844175762601
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,16384,0.08081422249476115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,10240,0.05470400055249532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,8192,0.04605955547756619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,5120,0.0063697778516345555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,7168,0.041900446017583214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,4096,0.005713777823580637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,3584,0.005599110904667113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,3072,0.005175999883148405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,65536,0.30134667290581596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,6144,0.037301331758499146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,5120,0.033167110549079046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,4096,0.028802666399214003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,2560,0.005370666583379109
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,3584,0.026664000418451097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,2048,0.004496888981925117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,1536,0.004113777644104428
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,1024,0.005617777920431561
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,3072,0.02426133387618595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,768,0.0046302220887608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,2560,0.021957332889238994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,512,0.003670222229427762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,2048,0.019738665885395475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,1536,0.017299556069903903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,1024,0.015040889382362366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,256,0.0027582223216692605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,128,0.0022524444179402455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,768,0.01421688828203413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,64,0.0021582223061058256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,32,0.002141333288616604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,512,0.012883555557992725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,256,0.011633777783976661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,65536,0.017799999978807237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,16384,0.007850666840871176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,128,0.01093066649304496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,64,0.010784889260927836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,12288,0.00702400008837382
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,32,0.010842666857772403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,10240,0.006637333167923822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,8192,0.006088888893524806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,16384,0.0803768899705675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,7168,0.005710222240951326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,12288,0.06334044535954793
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,8192,0.04583022329542372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,10240,0.05439733465512594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,6144,0.005229333208666907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,5120,0.005126222140259213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,4096,0.004615111069546806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,6144,0.037215110328462385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,3584,0.004867555366622077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,7168,0.04151111178927951
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,65536,0.2898702091640896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,5120,0.03279377685652839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,3072,0.00444444434510337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,2560,0.0047022220161226065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,4096,0.028463999430338543
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,2048,0.004025777594910728
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,1536,0.0038240001433425476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,3584,0.026230222649044458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,2560,0.021715554926130507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,1024,0.003539555602603488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,3072,0.024159111910396155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,2048,0.01942044496536255
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,768,0.004527111020353106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,512,0.003597333199448056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,1536,0.01720355616675483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,256,0.002663111107216941
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,128,0.0022400000856982335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,1024,0.01499288943078783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,64,0.0021315556433465746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,512,0.012786666552225748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,32,0.0021084443562560612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,768,0.01389422184891171
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,256,0.01145600030819575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,128,0.010985777609878115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,65536,0.016325333052211337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,16384,0.0069413334131240845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,64,0.01072533345884747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,12288,0.0058711109062035876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,32,0.010637332995732626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,10240,0.005946666830115848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,8192,0.005479111025730769
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,7168,0.004911111046870549
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,16384,0.07959289020962186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,12288,0.0626044472058614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,10240,0.054062220785352916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,6144,0.0047733332547876565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,8192,0.04528177777926127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,5120,0.004828444371620814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,4096,0.004273777620659934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,7168,0.04114044374889798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,3584,0.004297777596447202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,65536,0.2920728789435493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,6144,0.036879112323125206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,5120,0.03251911203066508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,3072,0.004182222402758068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,4096,0.028168890211317275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,2560,0.004431999805900785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,2048,0.003726222034957674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,3584,0.025981333520677354
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,1536,0.003654222107595868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,3072,0.02397777802414364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,1024,0.003456888927353753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,2560,0.021592888567182753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,768,0.0035377778112888336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,2048,0.019347555107540555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,512,0.003597333199448056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,1536,0.017059556312031217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,256,0.0028080000645584534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,128,0.002248888835310936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,1024,0.01480888823668162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,64,0.002116444417172008
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,768,0.013889777991506787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,32,0.0021155555215146807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,512,0.01262399968173769
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,256,0.011596444580290051
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,128,0.010786666638321347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,64,0.01073155552148819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,32,0.010683555569913654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,10240,1.2897617552015517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,12288,1.5828399658203125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,8192,1.661906666225857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,8192,0.9808195961846246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,16384,2.0564986334906683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,10240,2.0503565470377603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,12288,2.7150586446126304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,7168,0.8779084417555066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,7168,1.4465342627631292
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,6144,0.7357600000169543
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,5120,0.6053573290506998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,6144,1.285680029127333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,5120,1.0463199615478516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,16384,3.256630155775282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,4096,0.45640357335408527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,3584,0.4041573206583659
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,2560,0.30444267061021596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,3584,0.7434906429714627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,2560,0.5408550898234049
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,2048,0.4388888941870795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,2048,0.2553155687120226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,4096,0.93414306640625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,3072,0.7127751244439019
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,3072,0.3802079889509413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,1536,0.20895555284288195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,1536,0.3367919921875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,768,0.12001510461171468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,256,0.04720888866318596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,768,0.19210310777028403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,512,0.11736888355678982
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,1024,0.15954221619500056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,64,0.019698666201697457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,128,0.09944444232516819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,256,0.09886133008533055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,1024,0.25089867909749347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,64,0.09588888618681167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,32,0.01144977741771274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,128,0.02678311036692725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,512,0.10516799820794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,32,0.09923822349972194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,12288,0.4408542315165202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,16384,0.8542630937364367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,16384,0.5685928662618002
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,12288,0.7858977847629123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,10240,0.5704799758063422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,10240,0.3758266766866048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,7168,0.24702133072747123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,8192,0.29886311954922146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,8192,0.491579532623291
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,7168,0.40886934598286945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,5120,0.2683466540442573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,5120,0.17612000306447348
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,6144,0.36072622405158145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,4096,0.21661333243052164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,6144,0.23221778869628906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,3584,0.18897510899437797
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,65536,2.012939453125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,3584,0.1139777766333686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,4096,0.14393777317470974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,2560,0.14662133322821722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,3072,0.09748978084988064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,2048,0.10559822453392877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,1536,0.08462222417195638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,1024,0.05454044540723165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,768,0.041780445310804576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,3072,0.17808800273471406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,2560,0.09124533335367839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,1536,0.0625822212960985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,512,0.01495733360449473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,2048,0.07661600245369805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,256,0.007443555527263218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,512,0.024118221468395654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,768,0.027978665298885767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,64,0.004535111288229625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,1024,0.047819554805755615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,128,0.02072355482313368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,32,0.0037706668178240457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,32,0.019523556033770244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,256,0.022428444690174524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,64,0.020026667250527274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,128,0.00554933316177792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,65536,3.275922563340929
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,16384,0.41105156474643284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,16384,0.6436044375101725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,12288,0.32042132483588326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,10240,0.2639395660824246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,10240,0.3980986542171902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,12288,0.5291742218865289
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,8192,0.323616001341078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,8192,0.2108302248848809
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,7168,0.2924391163720025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,6144,0.25651733080546063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,6144,0.1507200002670288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,7168,0.19723021984100342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,5120,0.21109244558546278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,4096,0.16951378186543783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,4096,0.09730133083131577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,65536,1.4386862648857965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,5120,0.13017955091264513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,3584,0.09526222281985813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,3584,0.15618756082322863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,3072,0.1361466646194458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,2560,0.11214044358995225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,1536,0.061815996964772545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,2048,0.08169866932762994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,2048,0.061596446567111544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,768,0.03057866626315647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,1024,0.04725955592261421
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,1536,0.04946488804287381
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,2560,0.07139377461539374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,1024,0.026534222894244727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,3072,0.0837528904279073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,768,0.02419822249147627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,128,0.00518488883972168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,256,0.00702400008837382
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,512,0.01074399964676963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,512,0.02221244408024682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,64,0.00416711096962293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,32,0.003118222165438864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,256,0.01963555481698778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,128,0.01901155544651879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,32,0.018181333939234417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,64,0.018862222631772358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,65536,2.4701591067843967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,12288,0.43342312177022296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,12288,0.26559556855095756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,16384,0.37330132060580784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,16384,0.5428293546040853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,10240,0.34246489736768937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,8192,0.27923109796312123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,7168,0.23641067081027559
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,8192,0.1811457739935981
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,6144,0.11832355128394233
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,10240,0.2365057733323839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,6144,0.2144168880250719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,5120,0.1859217749701606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,65536,1.3409644232855902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,7168,0.15221333503723145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,4096,0.08259289132224189
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,4096,0.14804000324673122
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,3584,0.12917155689663357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,5120,0.10886666509840225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,2560,0.08364266819424099
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,3072,0.06561599837409125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,3072,0.1115564505259196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,3584,0.08086933030022515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,2048,0.07566133472654554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,1024,0.035391112168629967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,2048,0.048707554737726845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,768,0.014176888598336114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,1536,0.036526223023732506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,65536,2.0626470777723527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,1024,0.024907555845048692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,2560,0.06111199988259209
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,1536,0.056840889983707
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,512,0.0107351111041175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,128,0.005136000199450387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,64,0.0037760001917680106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,128,0.018274666534529794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,32,0.0034471110751231513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,768,0.023038221730126276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,64,0.017722666263580322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,256,0.019077334139082167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,256,0.007653333246707916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,512,0.021570665968788996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,32,0.017852443787786696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,16384,0.2782506677839491
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,10240,0.2640266683366564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,16384,0.4201671017540826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,12288,0.22523289256625703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,10240,0.17786488268110487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,12288,0.3871902094946967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,8192,0.21768000390794542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,8192,0.12262933784061009
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,7168,0.1925804482565986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,7168,0.11603377925025092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,6144,0.1015617781215244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,6144,0.1808160013622708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,4096,0.10450666480594212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,65536,0.9812631607055664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,5120,0.1459946632385254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,3072,0.07820800277921888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,5120,0.08354044622845119
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,4096,0.07154666715198092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,2560,0.06562666760550605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,3584,0.10156000322765774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,3584,0.05863110886679756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,2048,0.05340622199906243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,1024,0.013724444641007317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,1536,0.04072711202833388
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,3072,0.056399113602108426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,768,0.010320888625250923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,1024,0.01921777758333418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,1536,0.0218577782313029
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,2048,0.038344889879226685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,512,0.00777422222826216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,256,0.005351999981535806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,768,0.01779111060831282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,2560,0.04575466778543261
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,128,0.0041928887367248535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,512,0.016639999217457242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,65536,1.6322142283121746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,32,0.0032231110251612137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,256,0.015503999259736804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,64,0.003245333416594399
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,128,0.014458666245142618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,32,0.014039999908871122
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,64,0.01389955480893453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,16384,0.3735022279951308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,12288,0.30752621756659615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,12288,0.17838399940066865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,10240,0.23330844773186576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,10240,0.13650933901468912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,16384,0.24813956684536406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,7168,0.1623422172334459
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,8192,0.1912551058663262
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,6144,0.1413333283530341
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,8192,0.1047262218263414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,7168,0.09254933065838283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,6144,0.08215733369191487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,5120,0.11391733752356635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,3584,0.08906488948398167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,3584,0.05251377820968628
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,5120,0.07665955358081393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,65536,0.9593715667724609
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,4096,0.06314844555324979
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,3072,0.06933244731691149
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,2560,0.05772888660430908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,2048,0.04721422327889336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,1536,0.03612711032231649
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,2560,0.040404445595211454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,1024,0.011221333510345884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,2048,0.023970666858885024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,768,0.009647111097971598
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,4096,0.10273866520987616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,65536,1.4570720460679796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,1536,0.020596444606781006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,3072,0.04660444458325704
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,512,0.007264888948864407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,256,0.005141333159473208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,128,0.004159111115667555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,64,0.003346666693687439
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,1024,0.018195556269751657
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,512,0.015843555331230164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,32,0.003141333245568805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,128,0.013808888693650564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,768,0.016992888516849942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,64,0.013607111242082385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,256,0.014769777655601501
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,32,0.01330844478474723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,12288,0.1376968887117174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,16384,0.2023777829276191
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,16384,0.3246808846791585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,12288,0.26876621776156956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,10240,0.2028026713265313
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,8192,0.16243378321329752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,10240,0.10989866654078166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,7168,0.1384177737765842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,7168,0.07842844724655151
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,8192,0.0899697807100084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,6144,0.12209422058529323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,5120,0.09940889146592881
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,4096,0.07956088913811578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,5120,0.06008177995681763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,3584,0.06840888659159343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,3072,0.059847997294531934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,6144,0.06980355580647786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,3584,0.04541866646872627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,2560,0.049785776270760425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,65536,0.8197893566555448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,2048,0.040793776512145996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,2560,0.029920889271630183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,1536,0.03143822153409322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,2048,0.021760889225535925
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,4096,0.054745776785744556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,768,0.008864888714419471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,1536,0.019868445065286424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,512,0.006763555523422029
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,256,0.004882666799757215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,65536,1.2600666681925456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,768,0.016387555334303115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,1024,0.017622222503026325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,512,0.01552088889810774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,128,0.0038373333712418876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,3072,0.043840001026789345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,1024,0.010319111247857412
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,64,0.0031884445084465873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,64,0.01311822235584259
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,128,0.012999110751681857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,32,0.01313511116637124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,256,0.01406311160988278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,32,0.0027751111321979095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,12288,0.11455999480353461
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,16384,0.1700062221950955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,12288,0.2219182252883911
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,16384,0.28049156400892467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,10240,0.19262933731079102
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,8192,0.1391528844833374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,10240,0.10326133171717326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,7168,0.07005155748791166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,8192,0.08029955625534058
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,6144,0.11528444290161133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,7168,0.11982488632202148
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,4096,0.070360885726081
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,3584,0.05676888757281833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,5120,0.09735644525951809
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,4096,0.04831022355291578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,65536,0.6529315312703451
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,3072,0.05011644297175937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,6144,0.0621431138780382
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,2560,0.041823999749289624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,3584,0.04100711146990458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,5120,0.057095110416412354
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,2048,0.03389422098795573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,1536,0.013734222286277346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,65536,1.058452394273546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,1024,0.010144000252087912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,768,0.008450667063395182
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,512,0.006565333240562015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,2048,0.021185777253574792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,1536,0.019623999794324238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,768,0.0162791113058726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,256,0.0052800000541739995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,3072,0.030883553955290053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,512,0.014857777290874057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,128,0.0036026665733920205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,64,0.002797333316670524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,2560,0.0236453331179089
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,256,0.013999111122555204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,64,0.012626666989591388
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,1024,0.017166222135225933
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,128,0.013271999855836233
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,32,0.012549333274364471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,32,0.0026391111314296722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,16384,0.22269778781467012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,16384,0.1266782283782959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,12288,0.17514577176835802
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,12288,0.09664533535639445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,10240,0.14956355094909668
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,8192,0.1063271098666721
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,10240,0.08249155680338542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,7168,0.09305066532558864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,8192,0.0684115555551317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,6144,0.08114755815929837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,65536,0.5536853472391764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,5120,0.06806221935484144
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,6144,0.05387466814782885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,7168,0.06405422422620985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,3584,0.049547556373808116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,5120,0.04750666684574551
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,3584,0.026883555783165827
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,3072,0.04270222120814853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,4096,0.055868446826934814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,65536,0.8604586919148763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,2560,0.032532443602879844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,1536,0.01108977778090371
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,2560,0.022493332624435425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,2048,0.02046133412255181
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,1024,0.008254222571849823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,2048,0.01420177850458357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,768,0.006830222076839871
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,1536,0.01851466629240248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,3072,0.024816888901922438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,512,0.007039111521508958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,4096,0.039000001218583845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,1024,0.016089777151743572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,128,0.003346666693687439
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,64,0.0026293332791990707
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,256,0.004928888960017098
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,256,0.012864000267452665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,512,0.014177777700954013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,32,0.002467555511328909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,768,0.01512977812025282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,128,0.012145777543385824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,32,0.011769777370823754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,64,0.011955555942323474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,12288,0.1504302157296075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,16384,0.19738399982452393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,10240,0.11879466639624701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,12288,0.08827910820643108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,16384,0.12417422400580512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,8192,0.09550488657421535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,8192,0.06226399872038099
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,10240,0.07597333192825317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,7168,0.08382222387525771
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,7168,0.05551022291183472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,6144,0.07277333074145846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,65536,0.5120035807291666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,5120,0.06020355224609375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,65536,0.7550026575724283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,4096,0.028632889191309612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,5120,0.043776889642079674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,3584,0.043882668018341064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,4096,0.04956000049908956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,6144,0.04957866668701172
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,3072,0.0336151123046875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,2048,0.012248888611793518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,3584,0.02665688925319248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,2560,0.02754222187731001
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,1536,0.010112000008424124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,2048,0.020435555113686454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,1536,0.01796533332930671
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,3072,0.024414221445719402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,768,0.006487999939256244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,1024,0.0075493330756823225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,512,0.00546488869521353
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,2560,0.022307554880777996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,256,0.0046302220887608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,128,0.003195555466744635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,768,0.015187554889255099
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,64,0.0027582223216692605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,512,0.013752889302041797
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,32,0.00240622212489446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,1024,0.015951999359660678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,128,0.011669333610269757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,64,0.011419555379284753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,256,0.012522666818565793
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,32,0.011692444483439127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,12288,0.1208462185329861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,16384,0.16852800051371256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,12288,0.08016177680757311
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,10240,0.10152000188827515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,16384,0.10497866736518012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,10240,0.06941511233647664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,8192,0.08300977945327759
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,7168,0.07249333461125691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,6144,0.06342044141557482
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,8192,0.057542224725087486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,7168,0.05106844504674276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,5120,0.053879999452167086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,4096,0.04300089014901055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,65536,0.46898402108086484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,65536,0.6551609039306641
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,6144,0.04568799999025133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,5120,0.03306311037805345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,4096,0.028572443458769057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,2560,0.013841778039932251
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,3072,0.0335457788573371
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,3584,0.038273778226640486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,2048,0.010847110715177325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,3584,0.026331555512216356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,1536,0.009112888740168678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,2560,0.02200266718864441
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,2048,0.0196426659822464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,1024,0.006897777732875612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,768,0.00599911105301645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,1536,0.017675555414623685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,256,0.004315555509593752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,512,0.005000888887378904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,1024,0.015604444675975375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,768,0.01464355488618215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,64,0.0024871110088295406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,3072,0.024101333485709295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,512,0.013571555415789286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,128,0.011404444773991903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,32,0.0023617777559492323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,256,0.012251555919647217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,128,0.0030737777964936364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,64,0.01128888875246048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,32,0.0111191107167138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,12288,0.09914222028520371
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,16384,0.13129244910346136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,16384,0.09519377681944106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,12288,0.07232889201905993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,8192,0.06822755601671007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,10240,0.0940284464094374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,10240,0.06331288814544678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,8192,0.05239199929767185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,7168,0.06022933456632826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,7168,0.043599112166298755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,6144,0.052188442813025586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,65536,0.44160623020595974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,65536,0.5330053435431586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,5120,0.04473511046833462
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,4096,0.03658577799797058
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,5120,0.03253422180811564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,3584,0.026287110315428838
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,6144,0.03640977872742547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,3584,0.03151999910672506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,2560,0.019557333654827543
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,3072,0.023090667194790308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,2048,0.015159999330838522
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,4096,0.028198222319285076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,1536,0.012833777401182385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,1024,0.009015999734401703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,2048,0.019507555498017203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,1536,0.01745244363943736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,512,0.005787555542257097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,1024,0.015422221687104968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,3072,0.024005333582560223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,256,0.0042035554846127825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,768,0.007528000407748752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,2560,0.021960000197092693
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,128,0.003192000091075897
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,768,0.014389332797792224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,512,0.013232889274756113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,64,0.0023839999404218462
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,32,0.002263111165828175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,256,0.012108444339699216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,32,0.010955555571450127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,128,0.011210666762457954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,64,0.011130666567219628
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,16384,0.10463911294937134
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,16384,0.08734666638904148
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,12288,0.08010222514470418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,12288,0.06540444162156847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,10240,0.06746666961246066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,8192,0.055092444022496544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,7168,0.048693332407209605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,10240,0.05717866950564914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,6144,0.04244355691803826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,65536,0.4340097904205322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,5120,0.036256889502207436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,8192,0.046203556987974376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,7168,0.040503111150529646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,65536,0.4126568900214301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,6144,0.036305778556399874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,4096,0.015451555450757345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,3584,0.012335999972290464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,3072,0.02167555524243249
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,2560,0.018444443742434185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,4096,0.028047998746236164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,2048,0.014557333456145393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,3584,0.025979555315441553
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,1536,0.01203644441233741
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,2560,0.02145333256986406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,1024,0.008656889200210571
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,5120,0.03207022282812331
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,2048,0.019254222512245178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,1536,0.017249777913093567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,768,0.007121777368916406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,512,0.00544888898730278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,3072,0.023981332778930664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,1024,0.015242666006088257
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,128,0.003027555636233754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,64,0.002256888896226883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,768,0.014246222045686511
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,256,0.004032000071472592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,512,0.01297511077589459
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,32,0.0021671110557185277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,128,0.011143111520343356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,64,0.010793777803579966
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,32,0.010788444843557147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,16384,0.08037244611316256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,256,0.011864888999197217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,12288,0.06184977955288357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,10240,0.052080889542897545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,16384,0.0794302225112915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,8192,0.042749332057105176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,12288,0.061591108640034996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,65536,0.3326382107204861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,10240,0.05277599891026815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,7168,0.037816888756222196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,8192,0.04456177684995863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,6144,0.03325600094265408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,5120,0.013608000344700284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,4096,0.011031111081441244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,65536,0.3747653431362576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,6144,0.03629600008328756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,5120,0.032207998964521624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,7168,0.04045955671204461
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,4096,0.028047998746236164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,2560,0.01721600029203627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,3584,0.025860443711280823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,2048,0.014018666413095264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,3072,0.010092444717884064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,1536,0.011174221833546957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,2560,0.021382222572962444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,1024,0.008189332981904348
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,768,0.006847111301289664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,3584,0.010447111394670276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,1024,0.015142222245534262
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,3072,0.023690667417314317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,512,0.0052488889131281115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,1536,0.01734755602147844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,768,0.014064000712500678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,256,0.0037813331517908308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,128,0.0028568889117903183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,64,0.002340444467133946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,2048,0.019345778557989333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,32,0.002102222293615341
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,512,0.012913777596420713
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,256,0.011740444435013665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,128,0.011143999795118967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,64,0.010810666614108615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,32,0.0107351111041175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,16384,0.060963557826148145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,12288,0.0424177779091729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,10240,0.03604711095492045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,8192,0.017704889178276062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,12288,0.06102133459515042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,65536,0.23458311292860243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,16384,0.07775911357667711
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,7168,0.013947554760509066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,6144,0.012705778082211813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,8192,0.04454400142033895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,5120,0.010213333699438307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,7168,0.040439999765819974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,4096,0.009108444054921469
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,6144,0.03628177775277032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,65536,0.32675645086500377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,10240,0.05273422267701891
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,3584,0.018391110830836825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,3072,0.01608888970481025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,2560,0.013825777504179211
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,3584,0.025750221477614507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,2048,0.011371555427710215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,2560,0.02144977781507704
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,1536,0.009126221968068017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,4096,0.027978665298885767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,5120,0.03223199976815118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,2048,0.0191600008143319
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,1024,0.006688888702127669
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,768,0.005522666705979242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,3072,0.02367555598417918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,1536,0.01716444392999013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,256,0.003144888828198115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,512,0.004296888907750447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,128,0.002526222210791376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,768,0.014175110393100314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,1024,0.015025777949227227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,512,0.0129439996348487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,64,0.0021262222694026097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,256,0.011746666497654386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,32,0.0018835556175973679
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,128,0.011108444796668159
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,16384,0.04281511240535312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,64,0.010760000182522668
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,32,0.010690666735172272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,12288,0.03013155526585049
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,10240,0.0129120002190272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,16384,0.07771466837988959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,65536,0.17395288414425322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,8192,0.012368888490729861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,7168,0.011685333318180509
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,10240,0.05271822214126587
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,12288,0.06090133057700264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,6144,0.010693333215183683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,5120,0.015063999427689446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,7168,0.04035377833578322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,8192,0.044481777482562594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,4096,0.012531555361217923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,65536,0.30231467882792157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,6144,0.03623022304640876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,5120,0.031968000862333514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,4096,0.027832888894610937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,2560,0.008126222425036961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,3072,0.008386666576067606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,3584,0.011595555477672152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,3584,0.02569244470861223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,2048,0.01128266668981976
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,1536,0.008994667066468133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,2560,0.021290666527218286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,768,0.005411555369695027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,1024,0.006555555595291986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,1536,0.01706755492422316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,2048,0.019055111540688407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,3072,0.023808000816239253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,512,0.0042275554604000514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,1024,0.014952000644471912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,256,0.003107555624511507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,128,0.002422222246726354
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,64,0.0020595555090241963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,768,0.013991999957296582
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,32,0.002230222233467632
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,512,0.012802666260136498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,256,0.011708444191349877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,128,0.011090666883521609
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,64,0.010832000109884473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,16384,0.017639999588330586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,32,0.010688888529936472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,12288,0.020690666304694284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,65536,0.10606311427222358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,10240,0.01759111053413815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,8192,0.014504889647165934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,16384,0.07755555709203084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,12288,0.06085333559248182
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,10240,0.052700443400277026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,6144,0.0125928885406918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,5120,0.011019555230935415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,7168,0.040417777167426214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,65536,0.28503910700480145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,4096,0.009333333207501305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,6144,0.0361751119295756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,8192,0.04435733291837904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,5120,0.03201155530081855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,3072,0.008316444853941599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,2560,0.007599111232492659
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,7168,0.0140666663646698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,3584,0.00908799966176351
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,2048,0.006877333339717653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,3584,0.02572622232966953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,4096,0.027768888407283362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,3072,0.023565333750512864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,1536,0.008773333496517604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,2560,0.02128444446457757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,1024,0.006446222050322427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,768,0.005274666680230035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,512,0.0041253334946102565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,2048,0.01905955539809333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,256,0.002969777832428614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,1536,0.017055999901559617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,1024,0.014988443917698331
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,768,0.0140328887436125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,128,0.002385777731736501
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,512,0.01276622215906779
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,64,0.001983111103375753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,32,0.0021760000122918025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,256,0.011643555429246692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,65536,0.052592890130149
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,64,0.010836444795131683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,16384,0.014395554860432943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,128,0.010889777706729041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,32,0.010633777413103314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,12288,0.01205599970287747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,8192,0.009304888546466827
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,10240,0.010802666346232096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,16384,0.07739022043016222
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,7168,0.009095999929640029
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,12288,0.0610355536142985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,6144,0.00812000036239624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,10240,0.05268266797065735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,8192,0.04447466797298855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,5120,0.007666666474607255
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,7168,0.04073066512743632
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,4096,0.006824888702895906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,65536,0.27818311585320366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,6144,0.03612622287538316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,3584,0.0069644442862934535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,5120,0.03199555476506551
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,4096,0.027648000253571406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,2560,0.0058311112225055695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,3072,0.0064071110553211635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,3584,0.02570666703912947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,2048,0.005600000007285013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,1536,0.004622222234805425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,3072,0.02347466680738661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,768,0.0047395556337303585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,1024,0.005719111197524601
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,2560,0.02127644419670105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,2048,0.019132445255915325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,1536,0.017024889588356018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,512,0.003724444243643019
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,256,0.00282577777074443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,128,0.002307555534773403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,768,0.013931555880440606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,1024,0.014863111906581454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,512,0.012741333080662621
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,64,0.002163555473089218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,32,0.0021333334346612296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,256,0.011591110792424945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,65536,0.029538667864269678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,16384,0.009567110902733272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,128,0.010951999988820819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,64,0.01086400035354826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,32,0.010643555886215635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,12288,0.008360000120268928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,10240,0.007738666401969061
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,8192,0.006993778049945831
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,7168,0.007088888850477006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,6144,0.0064693333374129395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,12288,0.060940444469451904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,10240,0.05278311173121134
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,8192,0.04439022143681844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,16384,0.07716621955235799
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,5120,0.005987555616431766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,7168,0.04014488723542955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,4096,0.005380444642570283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,3584,0.0052888890107472735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,65536,0.27779467900594074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,6144,0.036269333627488874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,3072,0.00519822206762102
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,4096,0.027639999985694885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,5120,0.03207822309599982
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,3584,0.02551199992497762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,2560,0.004772444566090901
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,2048,0.004654222064548069
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,1536,0.00407644444041782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,3072,0.023520888553725347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,1024,0.005514666852023866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,2560,0.02138311167558034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,768,0.004615111069546806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,512,0.003636444608370463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,2048,0.018936889039145578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,256,0.0027377777215507296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,1536,0.016964443855815463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,1024,0.014832889040311178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,128,0.0022577777918842104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,768,0.013912000589900546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,64,0.0021555556191338436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,32,0.0021555556191338436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,512,0.012808000048001608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,65536,0.014695111248228284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,256,0.01162577751610014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,128,0.011072888970375061
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,16384,0.007374222079912822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,64,0.010642666783597736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,32,0.01057866629627016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,12288,0.006853333363930385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,10240,0.006105777704053455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,8192,0.006169777777459886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,7168,0.005537777725193236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,6144,0.005190222213665645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,10240,0.052341332038243614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,16384,0.07699111435148451
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,12288,0.060553775893317334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,8192,0.04414933257632785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,7168,0.040037333965301514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,5120,0.004752000172932942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,4096,0.004889777965015835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,65536,0.2762942314147949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,3584,0.004643555730581284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,6144,0.03596266773011949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,3072,0.004385777645640903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,2560,0.004113777644104428
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,5120,0.03168177935812209
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,4096,0.027473777532577515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,2048,0.0038844446341196695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,3584,0.025432000557581585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,1536,0.003773333297835456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,1024,0.003565333369705412
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,3072,0.023265777362717524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,2560,0.02108000053299798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,768,0.00452533322903845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,2048,0.018792000081804063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,1536,0.016781333420011733
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,512,0.003580444388919406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,1024,0.014679110712475248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,256,0.002806222273243798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,768,0.013605333036846586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,128,0.0022640000614855024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,64,0.0021137777302000257
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,512,0.012667555775907306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,32,0.0021084443562560612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,256,0.011399110986126794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,128,0.01076533314254549
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,65536,0.016132444143295288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,64,0.010701333483060202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,16384,0.006801777829726537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,32,0.010640000303586325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,12288,0.005769777629110549
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,10240,0.005873777800136142
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,8192,0.005384888913896348
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,16384,0.07770666811201307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,12288,0.06098310814963447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,7168,0.005139555368158552
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,10240,0.05261244376500448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,5120,0.00444088876247406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,6144,0.0047128887640105355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,8192,0.044271111488342285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,65536,0.2796071105533176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,4096,0.0041724443435668945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,7168,0.040037333965301514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,3584,0.004830222162935468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,3072,0.0041368889311949415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,6144,0.036074668169021606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,5120,0.031664000617133245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,4096,0.027588443623648748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,2560,0.004421333471934001
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,2048,0.003714666598372989
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,3584,0.02535999980237749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,3072,0.023210666245884363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,1536,0.0035955554081334006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,1024,0.0034017778105205963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,2560,0.02104444470670488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,2048,0.018702222241295707
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,768,0.0034906665484110513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,512,0.003953777667548922
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,1536,0.016657777958446078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,256,0.0028026666906144884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,1024,0.014555555250909595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,128,0.0022177776942650476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,64,0.0021342221233579847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,768,0.013743110828929476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,32,0.0020871110674407748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,512,0.012587555580668978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,256,0.011331555744012197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,128,0.01072088877360026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,64,0.010645333263609143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,32,0.010571555958853828
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,10240,1.2816587024264865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,12288,1.5623920228746202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,8192,1.655136850145128
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,16384,2.047234641181098
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,10240,2.045501285129123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,8192,0.9715572992960612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,7168,0.8128506872389051
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,12288,2.707116444905599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,6144,0.6818488968743218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,7168,1.4431307050916884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,6144,1.277800030178494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,5120,0.5987662209404839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,5120,1.0410772959391277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,16384,3.249506632486979
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,4096,0.8410666783650717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,3584,0.40311111344231504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,4096,0.4844044579399957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,3584,0.7384755346510145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,3072,0.3516684373219808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,3072,0.6438204447428385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,2560,0.5363475481669108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,2048,0.43396888838873965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,2560,0.3233528931935628
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,1536,0.3619786633385553
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,2048,0.2732471095191108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,1024,0.23011554612053764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,768,0.17840266227722168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,512,0.1169466707441542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,256,0.052632000711229115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,768,0.12179822391933864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,512,0.10160622331831191
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,64,0.01126577787929111
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,128,0.09885066747665405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,1536,0.21421155664655897
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,32,0.010970667004585266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,256,0.09754133224487305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,1024,0.15747555096944174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,64,0.09527289205127293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,32,0.09788444307115342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,128,0.02661777701642778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,12288,0.40497599707709414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,16384,0.5112364557054307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,10240,0.5598275396558974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,10240,0.3403368790944417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,12288,0.7814675437079536
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,16384,0.8486062155829536
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,8192,0.4469911257425944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,8192,0.27900976604885525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,7168,0.36784889962938094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,7168,0.26981690194871694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,6144,0.3260817792680529
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,5120,0.26881776915656197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,5120,0.17384621832105848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,6144,0.2301173210144043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,4096,0.13050577375623915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,3584,0.18872800138261583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,4096,0.2358480029635959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,3584,0.11311022440592448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,2048,0.10523111290401882
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,65536,1.8436400095621746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,3072,0.17399822341071236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,1536,0.08018666505813599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,2048,0.07025600141949125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,3072,0.10618399911456639
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,1536,0.05741155809826321
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,2560,0.09143910805384318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,2560,0.14574933052062988
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,768,0.04668622215588888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,512,0.01979200045267741
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,256,0.0074942219588491656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,512,0.02439466615517934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,1024,0.06016533242331611
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,128,0.00545155546731419
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,1024,0.04643199841181437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,768,0.027657777070999146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,64,0.005017777697907554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,32,0.0037911112109820047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,256,0.022151110900772944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,128,0.02074399921629164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,64,0.020271110865804885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,32,0.02014222244421641
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,12288,0.31665510601467556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,16384,0.4348924424913194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,65536,3.2649582756890188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,10240,0.4069235589769151
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,12288,0.57987732357449
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,16384,0.6401555803087022
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,7168,0.28419287999471027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,10240,0.2608364423116048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,8192,0.3240639898512098
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,6144,0.25287644068400067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,8192,0.22700977325439453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,7168,0.1937911113103231
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,6144,0.1492533286412557
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,5120,0.20963111188676622
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,4096,0.10591644710964626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,3584,0.08734222253163655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,3072,0.12170399559868707
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,3584,0.15636000368330213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,65536,1.4099707073635523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,4096,0.18510399924384224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,5120,0.1295644442240397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,2560,0.10243378082911174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,1024,0.042598221037122935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,2048,0.08261511060926649
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,3072,0.08334489001168145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,2560,0.06658933560053508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,1536,0.06894044743643866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,2048,0.05515022079149882
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,768,0.03221866819593642
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,512,0.010660444696744284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,256,0.00702311098575592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,1536,0.046668443414900035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,1024,0.02574311031235589
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,128,0.005156444592608346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,64,0.004063111212518481
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,512,0.021551999780866835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,32,0.0032506665835777917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,768,0.02385777731736501
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,32,0.018429333964983623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,64,0.018393778138690524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,128,0.018481777773963082
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,256,0.01992266707950168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,65536,2.461393780178494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,10240,0.3403422302669949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,16384,0.34142133924696183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,12288,0.2824968761867947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,12288,0.43120090166727704
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,16384,0.59534576204088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,10240,0.21450222863091362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,8192,0.16639999548594156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,8192,0.26954756842719185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,7168,0.13831822077433267
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,7168,0.23568354712592232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,6144,0.2284773455725776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,5120,0.09897510872946845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,5120,0.16784266630808511
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,65536,1.195508427090115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,3584,0.12939733929104275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,4096,0.13140800264146593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,6144,0.127618670463562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,3072,0.09938400321536595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,4096,0.0897128913137648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,2560,0.09336800045437282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,3072,0.06499733527501424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,3584,0.07989866866005792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,2048,0.06781599919001262
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,1536,0.057761775122748486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,2560,0.06187822421391805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,1024,0.0354017780886756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,2048,0.04842666784922282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,1536,0.03485155436727736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,768,0.0140657772620519
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,512,0.01055555542310079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,256,0.0075146663520071245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,1024,0.024883555041419134
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,768,0.022898667388492163
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,64,0.003738666574160258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,128,0.01831999917825063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,128,0.005120888766315249
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,256,0.019105777144432068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,32,0.0029057777590221832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,32,0.01811288959450192
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,64,0.01737777723206414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,65536,2.055196550157335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,512,0.020945777495702107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,16384,0.4170862303839789
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,16384,0.2739937835269504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,12288,0.20546044243706596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,10240,0.2901848951975505
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,12288,0.38487911224365234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,10240,0.16313600540161133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,8192,0.21656533082326254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,8192,0.12046666940053304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,6144,0.09082577625910442
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,7168,0.11410400602552627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,7168,0.20501777860853407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,6144,0.1806995603773329
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,65536,0.972084469265408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,5120,0.13084622224171957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,3584,0.09032977951897515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,4096,0.0709164473745558
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,3072,0.07814133167266846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,4096,0.11704711119333903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,5120,0.08492889006932576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,3584,0.0639680027961731
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,3072,0.051691555314593844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,2048,0.05952799982494778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,2560,0.044736888673570424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,2048,0.036397334602144026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,1024,0.0148044443792767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,1536,0.04556266797913445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,768,0.009482666850090027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,65536,1.6149430804782445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,2560,0.0733671122127109
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,256,0.005520888914664586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,512,0.0075297777851422625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,128,0.004058666527271271
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,1536,0.021407110823525324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,512,0.016543999314308167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,768,0.017516444126764934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,256,0.015281778242852954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,128,0.014137778017255994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,1024,0.018664000762833487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,32,0.003195555466744635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,64,0.014081777797804939
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,32,0.0140337778462304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,64,0.0032222221295038858
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,16384,0.2387768957349989
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,10240,0.13144266605377197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,12288,0.17286488744947645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,12288,0.3066195646921794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,16384,0.3691653410593669
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,10240,0.2548062271542019
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,8192,0.18937955962287056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,8192,0.10350844595167373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,6144,0.13950844605763754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,7168,0.16199910640716553
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,5120,0.11342310905456543
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,7168,0.09261955817540486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,5120,0.06898044215308295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,4096,0.0917173359129164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,6144,0.08821866909662883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,3584,0.0796862244606018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,4096,0.057313779989878334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,3072,0.06926133235295613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,65536,0.8635075357225207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,3584,0.05215466684765286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,1536,0.03601600064171685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,2048,0.0469760000705719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,2048,0.0240000006225374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,1024,0.01054133309258355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,1536,0.020374221934212577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,768,0.009248889154858058
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,2560,0.06443822383880615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,512,0.00720355576939053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,3072,0.04944533440801832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,256,0.005033777819739448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,512,0.01573955516020457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,2560,0.03941244549221463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,1024,0.017829333742459614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,64,0.0030666666312350165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,768,0.01701866587003072
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,256,0.01475822263293796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,128,0.004060444318585926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,65536,1.4299821853637695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,32,0.0029475556479560006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,64,0.013287111288971372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,128,0.013917333549923368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,32,0.01406133340464698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,16384,0.19933688640594482
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,16384,0.3232159879472521
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,12288,0.26671910285949707
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,12288,0.13650400108761257
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,10240,0.20017688804202607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,10240,0.10873777998818292
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,7168,0.13937866687774658
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,7168,0.07766311036215888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,8192,0.08785777621799046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,5120,0.09840089082717896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,6144,0.1209075583351983
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,6144,0.06895377900865343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,65536,0.7395991219414605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,8192,0.17758133676317003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,4096,0.07925244172414143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,3584,0.06851910882525973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,5120,0.05981777773963081
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,4096,0.04988266693221199
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,3072,0.06676533487108019
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,3072,0.04023822148640951
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,2048,0.04071288969781663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,2560,0.027976890405019123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,1536,0.031122667921913996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,2048,0.021496888664033677
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,3584,0.04857422245873345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,1536,0.01937866707642873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,2560,0.04977777931425306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,65536,1.25182130601671
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,1024,0.017536888519922893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,512,0.006643555644485686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,768,0.008578666382365758
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,128,0.004003555410438114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,512,0.015417777829700045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,768,0.016255110502243042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,64,0.003023111157947116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,256,0.0048071112897661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,32,0.0029324444217814338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,256,0.014180445008807711
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,64,0.01311466677321328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,128,0.013186666700575085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,32,0.013418667018413544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,1024,0.010104888843165504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,12288,0.11355822616153294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,16384,0.16265332698822021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,16384,0.2770613299475776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,10240,0.1905724472469754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,12288,0.2396977742513021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,8192,0.07901510927412245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,10240,0.09313511186175877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,8192,0.13230044311947292
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,7168,0.11804621749454075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,6144,0.10273244645860459
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,5120,0.08688622050815159
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,65536,0.6339653333028158
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,6144,0.06124711036682129
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,7168,0.068996442688836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,4096,0.06593155860900879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,5120,0.05319999986224704
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,3072,0.0498488876554701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,3584,0.056687997447119824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,3584,0.0395431121190389
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,3072,0.028208888239330713
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,2048,0.03412355648146735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,1536,0.013572444518407186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,2048,0.020901333954599168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,1024,0.010076444182131026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,2560,0.04610044426388211
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,2560,0.02330844435426924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,512,0.006486222147941589
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,1536,0.019384000036451552
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,1024,0.016947555873129103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,4096,0.04437866806983948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,256,0.005176888985766305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,65536,1.0874462127685547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,128,0.0035413333939181436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,768,0.015969778100649517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,768,0.008467555873923832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,64,0.0029928889125585556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,32,0.0026213334252436957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,256,0.013935110635227628
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,128,0.012980444563759698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,64,0.01311466677321328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,32,0.013050666285885705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,512,0.015061333775520325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,16384,0.1243751049041748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,12288,0.17143644226921928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,16384,0.22164177894592285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,10240,0.13503289222717285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,8192,0.10594933562808567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,7168,0.0927635563744439
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,10240,0.08637422323226929
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,6144,0.08101511001586914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,7168,0.059642665916019015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,8192,0.06720266739527385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,12288,0.10327111350165473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,5120,0.06789688931571113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,65536,0.5390088823106554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,6144,0.052742222944895424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,3584,0.0492888887723287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,4096,0.05570933553907606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,5120,0.047112001313103564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,3072,0.03820888863669501
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,2560,0.03256266646915012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,4096,0.0381368895371755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,2048,0.013099555340078143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,3072,0.024347555306222703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,2560,0.02221333318286472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,65536,0.8269199795193143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,1024,0.00777511133088006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,1536,0.010607110957304636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,768,0.006650666809744305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,3584,0.02661955522166358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,256,0.004679111142953237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,512,0.006968888971540663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,768,0.015063999427689446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,1024,0.01609066625436147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,2048,0.020248000820477802
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,512,0.013991110854678683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,1536,0.018241778016090393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,64,0.0026435556097163092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,128,0.0033111110743549135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,256,0.013213333156373767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,128,0.0120000003112687
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,64,0.011776888536082374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,32,0.012062222593360476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,32,0.0025128889828920364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,12288,0.14476711220211452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,16384,0.11333510610792373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,16384,0.20040267043643525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,12288,0.08729155858357747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,8192,0.09502666526370579
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,10240,0.13184711668226454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,7168,0.08342044221030341
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,8192,0.062147557735443115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,7168,0.05492622322506375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,65536,0.4989964697096083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,5120,0.05999733342064751
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,6144,0.08131911357243855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,65536,0.7287119759453667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,6144,0.04892622099982368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,4096,0.04914044340451559
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,10240,0.0745591123898824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,5120,0.04243466589185926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,3584,0.04373866650793287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,3072,0.03369955552948846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,2560,0.028265777561399672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,1536,0.009919111099508073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,2560,0.021889777647124395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,4096,0.02859555681546529
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,1024,0.007432888779375289
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,3584,0.026037333740128413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,2048,0.011721777419249216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,1536,0.017845332622528076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,2048,0.01973511113060845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,768,0.0063279999627007385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,3072,0.02407466702991062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,512,0.00508266646001074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,256,0.004484444442722532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,64,0.00250133333934678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,1024,0.016185777054892648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,128,0.003121777748068174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,512,0.013768889009952545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,768,0.014679110712475248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,128,0.011635555161370171
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,256,0.012578667038016848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,32,0.002410666603181097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,64,0.011343999869293638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,32,0.011708444191349877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,12288,0.12127555741204156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,16384,0.1711031066046821
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,16384,0.1020337806807624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,10240,0.10137866603003608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,12288,0.07898222075568305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,8192,0.056743999322255455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,8192,0.08282666736178927
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,6144,0.0632408857345581
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,7168,0.07234222359127469
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,10240,0.06812622149785359
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,7168,0.05028355452749464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,5120,0.05386755532688565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,4096,0.04275911053021749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,6144,0.044941332605150014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,5120,0.03220444586541917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,65536,0.45458846622043186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,65536,0.6482551362779405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,4096,0.027931554449929133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,3584,0.0380657778845893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,3072,0.03315022256639268
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,2560,0.012921777864297232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,2048,0.010489777558379702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,1536,0.00868266655339135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,3584,0.02558311157756382
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,1024,0.006618666566080517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,3072,0.02368444369898902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,768,0.005738666488064661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,2560,0.021548445026079815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,1536,0.017664889494578045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,512,0.0046951112647851305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,1024,0.01552088889810774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,256,0.0041866666740841335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,2048,0.019554666346973844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,128,0.003000000077817175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,64,0.002704888789190186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,32,0.0023573332776625953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,256,0.01221777747074763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,64,0.011215111447705163
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,128,0.01126399967405531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,768,0.014548444085650973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,32,0.011471110913488599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,12288,0.099108444319831
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,16384,0.13189155525631374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,12288,0.07074311044481066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,16384,0.0935902198155721
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,10240,0.08371466398239136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,512,0.013379555609491138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,8192,0.06791110833485921
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,8192,0.05152533451716105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,65536,0.42495200369093156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,10240,0.06182666619618734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,65536,0.5277946790059408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,6144,0.05184088812934028
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,5120,0.04420622189839681
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,4096,0.03644266724586487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,7168,0.05981955263349745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,7168,0.04807466599676344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,6144,0.03579377796914842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,3072,0.022260444031821355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,4096,0.027657777070999146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,3584,0.027169777287377253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,5120,0.03201955556869507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,2560,0.018422222799725003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,3584,0.02565866708755493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,2048,0.014854222536087036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,1536,0.012040889097584618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,2560,0.02162133322821723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,768,0.007327111230956183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,1024,0.00885333369175593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,1536,0.017363554901546903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,512,0.005600888695981767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,2048,0.019366222951147292
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,3072,0.02343911098109351
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,256,0.004121777911980947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,128,0.003220444545149803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,64,0.002480000050531493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,512,0.013312000367376538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,768,0.014481777946154276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,256,0.012235555383894177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,1024,0.015322667029168872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,32,0.0022408889813555614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,64,0.010913777682516309
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,128,0.011163555913501315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,32,0.01131377783086565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,12288,0.08028266827265422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,16384,0.1045431097348531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,10240,0.0673111081123352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,16384,0.0859137773513794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,8192,0.05506844321886698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,7168,0.048584000931845776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,10240,0.056551112069023975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,8192,0.045379555887646146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,12288,0.0700826644897461
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,65536,0.42978665563795304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,6144,0.04233955674701267
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,5120,0.036072889963785805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,65536,0.4002231227027045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,4096,0.0125946667459276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,6144,0.03572355707486471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,3584,0.011678222152921887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,7168,0.039841777748531766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,3072,0.02130311065249973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,2560,0.017793777916166518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,5120,0.031606223848130964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,4096,0.027445332871543035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,3584,0.025421332981851365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,2048,0.014386667145623101
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,3072,0.023281777898470562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,2560,0.021136000752449036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,1024,0.00850666645500395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,768,0.0069644442862934535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,1536,0.011613333390818702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,512,0.005370666583379109
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,1024,0.015178667174445258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,768,0.013998222019937305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,128,0.003030222323205736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,256,0.0038853333228164246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,512,0.012929778132173749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,64,0.0022391111900409064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,256,0.01203733351495531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,2048,0.019147555033365887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,32,0.0021662221600612006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,1536,0.017088888419999015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,128,0.011008000208271874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,32,0.011125333607196808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,16384,0.0802746680047777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,12288,0.061639110247294106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,64,0.010805333654085795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,16384,0.0779635575082567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,10240,0.052016887399885386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,8192,0.04267288910018074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,65536,0.3285955588022868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,12288,0.06029155519273546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,7168,0.037809779246648155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,10240,0.05188888973659939
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,6144,0.03199733297030131
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,8192,0.04383822282155355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,5120,0.01221244451072481
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,65536,0.36005687713623047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,4096,0.010604444477293225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,3584,0.010198222266303169
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,7168,0.03965866565704346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,6144,0.03555733296606276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,4096,0.027519999278916255
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,5120,0.03157777918709649
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,3584,0.02530044482813941
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,2560,0.01669066647688548
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,2048,0.013733333183659447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,3072,0.02314844396379259
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,3072,0.009415999882751042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,1024,0.00793422261873881
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,2560,0.02106933295726776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,768,0.005416888743638992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,2048,0.019007111589113872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,512,0.005038222091065513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,1536,0.017096888687875535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,1024,0.014986667368147107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,1536,0.010956444674068026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,256,0.0036915557252036203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,768,0.014001778430408902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,128,0.002776888923512565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,64,0.0022053333620230355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,32,0.0019422223170598347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,512,0.013046222428480784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,256,0.011784888803958893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,128,0.010980444649855295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,64,0.010794666906197866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,32,0.011021333436171213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,12288,0.04232266545295715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,10240,0.0359866652223799
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,16384,0.07601688967810737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,8192,0.013183111117945777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,65536,0.2311697800954183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,12288,0.05982222159703573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,7168,0.013473778135246701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,10240,0.05176711082458496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,6144,0.012111110819710625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,65536,0.3080008824666341
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,5120,0.010816888676749336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,16384,0.0546960002846188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,8192,0.04363377888997396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,4096,0.008952889177534316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,6144,0.03557422094874912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,3584,0.018349332941903006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,7168,0.03987288806173537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,3072,0.016191111670600045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,2560,0.013770666387346057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,4096,0.027319111757808264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,3584,0.02516355613867442
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,5120,0.03151466780238681
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,2048,0.011295999917719098
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,3072,0.023160000642140705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,1536,0.009096889032257928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,1024,0.006653333289755716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,2560,0.021162667208247717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,768,0.005526222288608551
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,2048,0.019096889429622226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,512,0.004243555582231945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,1536,0.017027555240525138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,256,0.0031617778456873367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,1024,0.015093333191341825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,768,0.01388088862101237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,128,0.002593777866827117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,64,0.0020951111283567217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,512,0.012979555461141797
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,32,0.0019902222686343724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,256,0.011900444825490316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,128,0.010969777901967367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,64,0.010789333118332757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,16384,0.04276622335116068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,32,0.011016888750924004
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,12288,0.03314222229851617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,10240,0.012126222252845764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,65536,0.17083111074235705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,8192,0.0105608892109659
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,16384,0.07605333460701837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,7168,0.011179555621412067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,12288,0.06047111087375217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,10240,0.05182400014665392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,6144,0.010262221925788457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,65536,0.282004435857137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,8192,0.043543110291163124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,5120,0.009080000221729279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,4096,0.012059555285506778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,3584,0.01131199962562985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,7168,0.039623111486434937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,3072,0.008338666624493068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,5120,0.031494223409228854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,2560,0.008030222521887885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,4096,0.02718222141265869
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,6144,0.035530666510264076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,3584,0.02535466684235467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,1536,0.008925333619117737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,2048,0.006690666493442323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,3072,0.023120888405376013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,2560,0.021107556091414556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,1024,0.006487111250559489
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,768,0.005380444642570283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,2048,0.01881511178281572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,1536,0.016970665918456185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,512,0.0041928887367248535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,256,0.0030666666312350165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,128,0.00239644447962443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,1024,0.014797333214018079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,64,0.002024000096652243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,768,0.013818666338920593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,512,0.012776888906955719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,32,0.002226666650838322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,256,0.011680888632933298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,128,0.011136000355084738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,16384,0.017772444420390658
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,64,0.010784889260927836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,32,0.010992888775136737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,65536,0.10588266452153523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,12288,0.010377777947319878
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,10240,0.017089777522616915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,8192,0.014341332846217685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,16384,0.07562933365503947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,7168,0.013586666848924426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,6144,0.012144000165992312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,12288,0.05974133147133721
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,10240,0.05179644293255276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,8192,0.04366577665011088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,5120,0.010758221977286868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,7168,0.039577778842714094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,4096,0.006537777682145436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,3584,0.006217777729034424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,6144,0.03552711009979248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,3072,0.005872888697518243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,5120,0.031452443864610456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,2560,0.005608888963858287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,4096,0.027308444182078045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,2048,0.006773333168692059
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,3584,0.025103110406133864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,65536,0.2732488844129774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,1536,0.008773333496517604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,3072,0.023048889305856492
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,2560,0.021055999729368422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,1024,0.0058986664646201665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,768,0.0053031109273433685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,2048,0.01882755590809716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,512,0.00408000002304713
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,1536,0.016893333858913846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,256,0.0029599999801980127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,128,0.0023591110689772502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,1024,0.014893333117167154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,768,0.013737777868906656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,64,0.001993777851263682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,32,0.0021866667601797315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,512,0.012768000364303589
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,256,0.01163999984661738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,128,0.010924444430404238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,64,0.010738666686746808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,16384,0.013942221800486246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,12288,0.008298666940795051
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,32,0.011001778145631155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,65536,0.052501334084404834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,10240,0.01052177780204349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,8192,0.009081777599122789
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,7168,0.008820444345474243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,16384,0.0754951106177436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,12288,0.059771557648976646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,6144,0.007932444413503012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,10240,0.05171822177039253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,5120,0.007544889218277401
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,4096,0.00737955586777793
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,7168,0.039517333110173546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,8192,0.04361688759591845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,65536,0.27066755294799805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,6144,0.03547644284036424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,3584,0.0069164443347189165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,3072,0.006385777973466449
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,2560,0.0058115555180443665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,4096,0.027186666925748188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,2048,0.005608888963858287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,3584,0.02502311103873783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,3072,0.023199111223220825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,5120,0.0314044455687205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,1536,0.004593777573770947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,2560,0.020899555749363367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,1024,0.005625777774386936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,768,0.004750222381618288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,2048,0.018801778554916382
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,512,0.0037288889288902283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,1536,0.01702222228050232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,256,0.002850666642189026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,1024,0.014860444598727755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,768,0.013748444616794586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,128,0.002320000073975987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,512,0.012675555215941535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,64,0.0021537778278191886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,32,0.0021537778278191886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,256,0.011619555453459421
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,128,0.010955555571450127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,65536,0.02786488996611701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,16384,0.009343110852771336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,64,0.01068800025516086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,12288,0.008238222036096785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,32,0.010981332924630908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,10240,0.007693332930405934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,8192,0.0068915556702348925
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,16384,0.0751439995235867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,7168,0.007055111229419708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,12288,0.05970399909549289
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,10240,0.05192888776461283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,6144,0.0064497776329517365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,8192,0.04331022169854906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,5120,0.005943999936183293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,4096,0.005342222336265776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,3584,0.005290666802061929
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,65536,0.2704559961954753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,7168,0.03937866621547275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,5120,0.03169599837727017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,3072,0.005160000175237656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,4096,0.027075555589463975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,2560,0.004736888739797804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,6144,0.03567377726236979
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,2048,0.004635555462704765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,3584,0.024937777055634394
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,1536,0.004053333153327306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,1024,0.005504888792832692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,2560,0.021033777130974665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,3072,0.02313866714636485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,768,0.004569777597983678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,2048,0.018813333577579923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,512,0.003625777860482534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,256,0.0027102221631341507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,1536,0.017009778155220878
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,1024,0.014694222145610385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,128,0.0022400000856982335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,512,0.012757333616415659
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,64,0.0021671110557185277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,32,0.0021182222084866627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,256,0.011598221957683563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,768,0.013719999955760108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,128,0.010942222343550788
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,16384,0.007228444847795699
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,65536,0.014365333649847241
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,64,0.010690666735172272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,12288,0.006818666640255187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,32,0.011022222538789114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,10240,0.006081777728266186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,8192,0.005614222337802251
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,7168,0.005554666535721884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,16384,0.07537600066926745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,10240,0.05117155445946587
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,6144,0.005215999980767568
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,12288,0.05935021903779772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,8192,0.04328622089491951
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,7168,0.039114667309655085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,5120,0.0047235555118984645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,4096,0.004866666677925321
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,65536,0.26876356866624623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,6144,0.035232888327704534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,3584,0.00464088883664873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,3072,0.004345777961942884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,5120,0.031138665146297876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,4096,0.027052443888452318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,2048,0.0038693332009845306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,2560,0.004512888689835866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,3584,0.024680000212457445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,1536,0.0037360000941488477
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,1024,0.0035422220826148987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,3072,0.022808889547983806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,2560,0.020615999897321064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,768,0.004538666870858935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,2048,0.018727999594476488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,512,0.003587555554178026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,1536,0.016747555798954435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,256,0.0028515555378463534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,1024,0.014563555518786112
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,128,0.0022168887986077205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,768,0.013535999589496188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,64,0.002135111019015312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,512,0.012500444220172035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,32,0.0020853332761261198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,256,0.011528000235557556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,65536,0.016131555040677387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,128,0.010753778119881948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,16384,0.006796444455782573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,64,0.010658666491508484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,12288,0.00573511090543535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,32,0.010838222172525195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,10240,0.0057955553962124725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,8192,0.0053102220926019884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,16384,0.07536177502738105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,7168,0.00509688879052798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,12288,0.059446222252315946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,10240,0.05113600028885735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,6144,0.004613333278232151
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,5120,0.004414222306675381
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,8192,0.04310577776696947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,65536,0.26938843727111816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,4096,0.004150222159094281
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,7168,0.03920800156063504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,6144,0.03511555658446418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,3584,0.004859555512666702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,5120,0.031000001562966242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,3072,0.0041128889554076726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,4096,0.026887110537952844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,2560,0.004419555680619346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,2048,0.00369777778784434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,1536,0.003647110942337248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,3584,0.02459200057718489
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,3072,0.022669333550665114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,1024,0.0034044444974925783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,2560,0.020483555065260995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,768,0.005048888838953442
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,2048,0.018592000007629395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,512,0.003927111211750242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,1536,0.016703110602166917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,256,0.002790222151411904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,1024,0.014471999473041959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,128,0.002228444442152977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,768,0.013440888788965015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,64,0.0021057778762446507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,512,0.012479999827014076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,32,0.0021182222084866627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,256,0.011388444238238864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,128,0.010724444356229572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,64,0.010674667027261523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,32,0.010817777779367236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,10240,1.2039786444769967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,12288,1.4532818264431422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,8192,1.6891964806450739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,16384,1.9068417019314234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,10240,2.0647964477539062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,8192,0.9293218188815646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,7168,0.788326210445828
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,12288,2.727925406561958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,5120,0.5382693078782823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,6144,0.7087030940585666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,7168,1.4375368754069011
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,6144,1.2747689353095162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,5120,1.0365848541259766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,16384,3.2825565338134766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,4096,0.46762042575412327
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,4096,0.8368044429355197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,3584,0.39522221353318954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,3072,0.3323102262285021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,2560,0.2840577761332194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,2560,0.5318693584865993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,1024,0.22528444396124947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,3584,0.8093466758728027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,1536,0.32982667287190753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,3072,0.7035608821445041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,2048,0.4703555636935764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,1536,0.19515644179450142
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,512,0.10572355323367649
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,768,0.10338933600319757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,512,0.09400177664226955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,2048,0.25329245461357963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,256,0.055946667989095054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,64,0.010582221878899468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,128,0.08659111128913032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,32,0.0068853331936730286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,256,0.08704533179601033
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,768,0.187882661819458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,1024,0.14030667146046957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,64,0.08850489060084026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,128,0.02514755560292138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,32,0.08769244617886013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,12288,0.39766044086880153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,10240,0.3294088840484619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,16384,0.5101848708258735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,10240,0.5152435302734375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,12288,0.7572533289591471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,16384,0.8443066808912488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,7168,0.22079643938276503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,8192,0.2522266705830892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,8192,0.46696000629001194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,7168,0.4065706729888916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,6144,0.360459566116333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,6144,0.20168799824184844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,5120,0.2647342152065701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,5120,0.1510337723626031
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,4096,0.23456266191270617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,3072,0.08865066369374593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,3584,0.20626666810777453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,3584,0.11301333374447292
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,4096,0.1292168829176161
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,3072,0.176728884379069
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,2560,0.13140177726745605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,2560,0.07592088646358915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,65536,1.7446550793117945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,1536,0.08875199821260239
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,1024,0.05453777644369337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,2048,0.06330221891403198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,1024,0.0355022218492296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,768,0.018428444862365723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,512,0.015046222342385186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,2048,0.11672800117068821
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,1536,0.05534755521350437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,768,0.04514133267932468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,256,0.00573333352804184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,64,0.0036764442920684814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,128,0.004657777647177379
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,256,0.0155395550860299
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,64,0.0140684445699056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,128,0.014479110638300577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,512,0.016744888491100736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,32,0.014114666316244336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,32,0.003572444534964032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,16384,0.39313244819641113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,12288,0.30566755930582684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,16384,0.6165351337856716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,65536,3.2552543216281467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,12288,0.5760880046420628
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,10240,0.3900275495317247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,8192,0.18639200263553193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,8192,0.3188390996721056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,10240,0.24963198767768013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,7168,0.2908693419562446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,6144,0.24389955732557508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,65536,1.3483617570665147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,5120,0.10795555512110393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,7168,0.16998933421240914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,5120,0.2126577827665541
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,6144,0.14176000489128962
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,4096,0.1573751105202569
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,3584,0.13386578030056423
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,3584,0.07751555575264825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,3072,0.11434933874342178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,3072,0.07458666960398357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,2048,0.07727466689215766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,4096,0.09694577587975396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,1024,0.0409733321931627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,2048,0.0487973325782352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,2560,0.10791911019219293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,768,0.03046311272515191
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,1536,0.04286311070124308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,512,0.007382222347789341
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,256,0.005335999859703912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,1024,0.01774666706720988
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,1536,0.06588000059127808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,512,0.015453333655993143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,2560,0.06338666545020209
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,128,0.004417777889304691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,64,0.003628444340493944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,32,0.003128000017669466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,256,0.014154665999942355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,64,0.013076444466908773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,128,0.013437333206335703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,768,0.01651733285850949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,32,0.012850667039553324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,65536,2.4208176930745444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,12288,0.23898755179511177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,10240,0.3236009014977349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,16384,0.31349865595499676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,10240,0.20825333065456816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,12288,0.42950932184855145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,8192,0.2676222324371338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,7168,0.22741243574354383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,8192,0.14622310797373453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,16384,0.5706239806281196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,7168,0.13723556200663248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,6144,0.2010115517510308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,5120,0.1636213329103258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,5120,0.09104710817337036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,3584,0.11232177416483562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,4096,0.14422667026519775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,3072,0.10537866751352946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,6144,0.11815555890401204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,4096,0.08244266774919298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,65536,1.142807960510254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,3072,0.05867822302712334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,2560,0.050811555650499135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,2048,0.0716186695628696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,2560,0.0971662203470866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,1024,0.034621334738201566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,2048,0.046599109967549644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,768,0.008700444466537898
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,1024,0.016866667403115165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,3584,0.07428266604741414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,512,0.006882666713661618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,768,0.015933333171738517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,1536,0.059614221254984535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,128,0.003951999876234266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,64,0.0031591111587153543
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,128,0.012830221818553077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,256,0.005131555514203178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,32,0.0029120000286234748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,256,0.013991110854678683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,64,0.013018666870064206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,32,0.012519111235936483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,1536,0.026175111532211304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,65536,2.0059244367811417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,512,0.014881778094503613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,16384,0.2660382323794895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,12288,0.19933688640594482
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,16384,0.4140888849894206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,12288,0.3781857755449083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,10240,0.2613191074795193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,10240,0.1620319949256049
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,8192,0.12437866793738471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,6144,0.1622355514102512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,7168,0.18461155891418457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,8192,0.23753865559895834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,7168,0.10655377970801459
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,65536,0.9005084567599826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,5120,0.13038133250342474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,6144,0.09230489201015896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,4096,0.11739999718136257
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,3072,0.07803110943900214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,3584,0.10143288638856675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,4096,0.07317866881688435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,5120,0.0875742236773173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,3584,0.06154488854938083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,2560,0.06564444303512573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,3072,0.05900888972812229
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,2048,0.053293334113227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,1536,0.04063377777735392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,1024,0.017232000827789307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,2560,0.048264887597825795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,768,0.009038222332795462
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,2048,0.040911999013688825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,512,0.005961777849329843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,65536,1.6076924006144206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,256,0.004608889006906086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,1024,0.02344266739156511
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,128,0.003911111089918348
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,1536,0.02565244502491421
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,512,0.0221742226017846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,32,0.0027919999427265594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,768,0.02278311053911845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,64,0.019298666053348117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,64,0.0029360000044107437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,128,0.019739554988013375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,256,0.020555555820465088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,32,0.0195777780479855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,12288,0.16825332906511095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,10240,0.22997689247131348
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,16384,0.23096444871690539
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,12288,0.304603550169203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,10240,0.13456532690260145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,8192,0.18882221645779082
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,7168,0.16004355748494467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,8192,0.10467644532521565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,16384,0.40615910953945583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,7168,0.0929608874850803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,6144,0.13935022883945042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,5120,0.07710578044255574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,5120,0.11251821782853867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,3584,0.07936622036827935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,4096,0.05962577793333265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,65536,0.8202764193216959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,4096,0.10142399867375691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,6144,0.08959111240175034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,2560,0.05700711409250895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,3072,0.06764177481333415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,3584,0.058372444576687284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,3072,0.04861244559288025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,1536,0.035437332259284124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,2048,0.051572445366117686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,768,0.007183999651008182
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,2048,0.03105777833196852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,1024,0.014228444960382251
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,512,0.006015111174848344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,1024,0.022259554929203455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,1536,0.023549333214759827
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,65536,1.4116079542371962
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,128,0.00351200004418691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,256,0.004385777645640903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,2560,0.04627822173966301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,512,0.020376000139448378
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,256,0.0188071115149392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,32,0.0026977778308921387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,64,0.0027760000278552375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,64,0.01850755512714386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,768,0.021382222572962444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,32,0.018192888961897958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,128,0.018417777286635507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,12288,0.1404515504837036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,12288,0.262119107776218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,10240,0.1129395564397176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,10240,0.19736799928877088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,16384,0.317434655295478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,16384,0.2129084401660495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,8192,0.15884089469909668
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,7168,0.13207466072506374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,8192,0.09078755643632676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,7168,0.08122933573193021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,6144,0.07848444249894884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,5120,0.09489866760041978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,6144,0.12965688440534803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,4096,0.07610666751861572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,65536,0.7058560053507487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,3072,0.05808977948294746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,3584,0.04812444580925835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,2560,0.04868977930810717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,3072,0.043918222188949585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,5120,0.0624471108118693
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,4096,0.053095112244288124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,2048,0.03992444607946608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,2560,0.033619556162092425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,1024,0.012435555458068848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,1536,0.030722667773564655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,3584,0.06722488668229845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,768,0.009891555541091496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,65536,1.2273404863145616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,512,0.005327111317051782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,1536,0.022229333718617756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,256,0.004072000169091755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,2048,0.023694222172101338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,128,0.0035626664757728577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,1024,0.02030666669209798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,768,0.019655999210145738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,256,0.017879999346203275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,32,0.0026888888743188647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,64,0.0026773332307736077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,128,0.017315554949972365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,64,0.017287110288937885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,512,0.019109333554903667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,12288,0.20229955514272055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,12288,0.11227822303771973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,16384,0.2681537734137641
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,16384,0.17160444789462617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,10240,0.16296622488233778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,10240,0.09371466769112481
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,7168,0.10723288853963216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,32,0.017091555727852713
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,8192,0.13057777616712782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,65536,0.5971715715196397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,8192,0.07737600141101413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,7168,0.06967910793092516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,5120,0.07783822218577068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,4096,0.06269155608283149
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,6144,0.09523022174835205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,3584,0.05554044577810499
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,4096,0.046132445335388184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,6144,0.06731022066540189
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,3584,0.04145688811937968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,2560,0.040609776973724365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,2048,0.03313688768280877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,5120,0.058661335044436984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,3072,0.0328693323665195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,65536,1.026759147644043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,1536,0.017619555195172627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,1024,0.012160888976520963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,3072,0.05380355649524265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,2048,0.02253066665596432
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,768,0.010045333041085137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,2560,0.023804444405767653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,1024,0.01976088848378923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,256,0.004011555678314633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,1536,0.021301334102948506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,512,0.005072888814740711
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,128,0.0032462223122517266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,768,0.019243554936514962
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,64,0.0029404444826973807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,512,0.01851111153761546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,32,0.002467555511328909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,128,0.016927111479971144
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,32,0.016798221402698092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,64,0.016903110676341586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,256,0.017330666383107502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,16384,0.20117689503563774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,12288,0.1627697812186347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,16384,0.11893510818481445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,10240,0.07756800121731229
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,10240,0.13596622149149576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,8192,0.10515199767218696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,12288,0.09157333109113906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,8192,0.06468711296717326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,65536,0.5013173421223959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,5120,0.06074044439527723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,7168,0.08462044265535142
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,6144,0.07467466592788696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,6144,0.05259822143448723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,4096,0.050251556767357715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,3584,0.04363110992643568
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,5120,0.0462408893638187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,7168,0.06397422154744466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,4096,0.037008888191647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,65536,0.8243333498636881
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,3072,0.03824266791343689
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,3072,0.023016888234350417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,1536,0.013241777817408243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,2560,0.02183377742767334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,2048,0.020842666427294414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,1024,0.008988444175985124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,2048,0.01756000022093455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,768,0.007703111403518253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,3584,0.024862221545643274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,512,0.0058053334554036455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,1024,0.018236445056067575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,256,0.004402666870090696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,128,0.0034506666577524612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,2560,0.03563111027081808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,768,0.01779022150569492
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,1536,0.019440889358520508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,512,0.01700444519519806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,256,0.015669332610236276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,32,0.0023697778168651792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,128,0.015213333898120456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,64,0.0026364444444576898
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,64,0.01511111193233066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,32,0.015207111835479736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,16384,0.10498488611645168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,10240,0.11428266101413304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,16384,0.18981688552432588
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,12288,0.08201066652933757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,12288,0.14192177189721003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,10240,0.06951200299792819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,8192,0.09259111351437038
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,8192,0.05851822429233127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,7168,0.07501688930723402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,65536,0.44841686884562176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,6144,0.06650400161743164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,7168,0.05318133367432488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,5120,0.05400088760587904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,4096,0.044237332211600415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,65536,0.7244204415215386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,6144,0.04765244325002035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,5120,0.0406968891620636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,3584,0.0387644436624315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,3072,0.03403288788265652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,2560,0.025412445267041523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,4096,0.024829333027203877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,1536,0.012430222498046027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,2048,0.01981511049800449
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,2048,0.014848000473446317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,3584,0.02314311100376977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,768,0.00742133292886946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,2560,0.020872889293564692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,3072,0.022082666556040447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,1536,0.018575999471876357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,512,0.005655999812814925
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,1024,0.009069333473841349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,768,0.01685688893000285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,256,0.004284444368547863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,128,0.0033475555893447665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,1024,0.017487110363112558
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,512,0.01626488897535536
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,64,0.002430222307642301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,256,0.014934221903483072
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,32,0.002415111081467734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,128,0.014688000082969666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,32,0.014368888404634265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,64,0.014578666951921252
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,16384,0.1610942284266154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,12288,0.11731999450259739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,10240,0.10336799754036798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,12288,0.0736480024125841
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,16384,0.09312711159388225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,8192,0.07976710796356201
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,10240,0.06218577755822075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,7168,0.06555289030075073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,8192,0.05249333381652832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,7168,0.04790488878885905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,5120,0.04795111219088236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,6144,0.05844089057710436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,6144,0.04314755400021871
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,5120,0.028951111767027114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,65536,0.6272720230950249
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,4096,0.03901422354910109
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,3584,0.034119112624062434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,3072,0.02661955522166358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,2560,0.019147555033365887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,4096,0.02355555527740055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,3584,0.0224897778696484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,2048,0.01475644442770216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,1024,0.008751110898123847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,65536,0.43491821818881565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,2560,0.0199946661790212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,2048,0.018983110785484314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,1536,0.01789955629242791
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,768,0.007297777467303806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,1536,0.012627555264366997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,3072,0.021370665894614324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,256,0.004023999803596073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,1024,0.01667555504375034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,128,0.00315022220214208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,64,0.00259911103381051
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,768,0.016159999701711867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,512,0.005519999812046687
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,512,0.015451555450757345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,256,0.014365333649847241
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,32,0.0022355556074115965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,64,0.013914666242069669
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,128,0.0141004439857271
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,32,0.013810666898886362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,16384,0.1308079957962036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,12288,0.09823644161224365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,16384,0.0808853308359782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,12288,0.0646044413248698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,10240,0.08360799815919663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,7168,0.056890666484832764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,8192,0.06744177473915948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,8192,0.0463048881954617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,10240,0.05444799860318502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,6144,0.05061422122849358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,5120,0.04195555713441637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,65536,0.3466951052347819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,4096,0.033933334880405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,6144,0.03031377659903632
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,65536,0.5226888656616211
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,7168,0.04193777839342753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,3584,0.028983109527164038
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,3072,0.02181600034236908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,4096,0.023237332701683044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,2560,0.018266666266653273
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,3584,0.022068444225523207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,5120,0.025335111551814612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,2048,0.014282666974597506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,1536,0.011807999677128263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,1024,0.008266666697131263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,2048,0.01869511107603709
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,768,0.007136888802051544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,1536,0.01756888959142897
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,3072,0.02068711154990726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,512,0.005333333379692501
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,256,0.00388355553150177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,2560,0.019736889335844252
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,768,0.015631111131774057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,128,0.0029191111938820947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,1024,0.016354666815863717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,512,0.015182221929232279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,32,0.0023031110564867654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,64,0.0022799999763568244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,128,0.013582222163677216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,256,0.013933334085676404
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,32,0.013291555146376291
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,16384,0.10441955592897202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,12288,0.07989511224958631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,16384,0.06932266553243001
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,64,0.013376000026861826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,12288,0.055584887663523354
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,10240,0.06718844175338745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,8192,0.05506933397716946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,10240,0.04684799909591675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,65536,0.29302578502231175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,6144,0.0407591097884708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,65536,0.424965328640408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,8192,0.04055555661519369
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,7168,0.029397332006030615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,5120,0.03465066684616937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,6144,0.0270142224099901
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,7168,0.046406222714318164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,4096,0.023751111494170293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,3584,0.020132443971104093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,3072,0.01696888936890496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,2560,0.014352000421947904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,4096,0.02236533330546485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,2048,0.011492444409264458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,3072,0.020049777295854356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,1536,0.009389333426952362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,5120,0.02478222217824724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,3584,0.021335111724005804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,1024,0.006950221955776215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,768,0.00574311117331187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,512,0.004513777792453766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,1536,0.01681511104106903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,2560,0.018917333748605516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,1024,0.01552977826860216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,128,0.0028106667515304354
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,768,0.015017777681350708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,256,0.003463111196955045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,2048,0.01795644395881229
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,64,0.0023182222826613323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,32,0.0021439999755885866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,256,0.013312000367376538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,512,0.014270222849316068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,128,0.012898666991127862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,64,0.012820444173283048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,32,0.012833777401182385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,12288,0.0614924430847168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,16384,0.08006399869918823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,16384,0.05834488736258613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,8192,0.04251555601755778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,10240,0.05189244283570183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,12288,0.04678044385380215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,10240,0.035916444328096174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,65536,0.3247759872012668
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,8192,0.03116977877087063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,65536,0.23646132151285806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,6144,0.036095112562179565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,7168,0.04212355613708496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,4096,0.02077244387732612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,5120,0.024417777856190998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,3584,0.018544000056054857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,5120,0.028947555356555518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,4096,0.02203377750184801
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,3584,0.020970667401949566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,6144,0.026699556244744196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,2560,0.013970666461520724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,1536,0.00920800036854214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,3072,0.016301333904266357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,3072,0.019688889384269714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,2048,0.011304000185595619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,2048,0.017431111799346078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,2560,0.018626666731304593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,1024,0.006743999818960826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,768,0.005580444302823808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,512,0.004323555363549126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,1536,0.01646311084429423
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,256,0.003228444399105178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,1024,0.01516177753607432
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,768,0.014634667171372307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,7168,0.02908888790342543
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,128,0.002642666714058982
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,512,0.014087110757827759
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,64,0.0021031111892726687
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,256,0.013082666529549493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,32,0.0019066666977273093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,128,0.0127368892232577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,16384,0.051073776351081006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,64,0.0125937776433097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,32,0.012473777764373355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,12288,0.039729777309629656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,10240,0.03346755438380771
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,8192,0.01832088828086853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,65536,0.18060710695054796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,65536,0.20756088362799752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,12288,0.04008533226119147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,16384,0.05215555429458618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,8192,0.03103733393881056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,7168,0.016604445046848722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,6144,0.014685332775115967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,10240,0.03556266758177016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,4096,0.010856000085671743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,7168,0.028930667373869155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,6144,0.02681066592534383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,5120,0.02437155610985226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,3072,0.010967111421955956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,3584,0.010418666733635796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,4096,0.02195377813445197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,2560,0.0096124443742964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,5120,0.012866666747464074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,2048,0.00830844458606508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,1536,0.008917333351241218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,3072,0.019837333096398246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,1024,0.006521777974234686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,2560,0.01843644513024224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,768,0.005403555515739653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,3584,0.020862221717834473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,1536,0.016347555650605094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,512,0.004187555362780889
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,256,0.003059555465976397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,1024,0.015173332558737861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,128,0.002440888848569658
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,64,0.0021137777302000257
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,768,0.014575999644067554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,512,0.013978666729397245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,256,0.012991999586423239
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,32,0.0018800000349680583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,128,0.012687111066447364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,64,0.012622222304344177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,16384,0.04257511099179586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,32,0.012416889270146688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,12288,0.030416889323128596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,10240,0.017582222819328308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,65536,0.16040089395311144
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,2048,0.017412443955739338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,16384,0.04841511117087471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,8192,0.014595554934607612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,7168,0.013920888304710388
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,12288,0.039862222141689725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,10240,0.035270220703548856
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,65536,0.15867643886142307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,6144,0.012680889003806643
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,5120,0.011062222222487131
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,8192,0.030737777551015217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,7168,0.028766221470303003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,4096,0.00941422250535753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,3584,0.009170666337013245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,6144,0.026612444056404963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,5120,0.024125332633654278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,3072,0.008401778009202745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,2560,0.007600888609886169
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,4096,0.021682666407691106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,2048,0.006858666737874349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,1536,0.008823110825485652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,3072,0.019519999623298645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,1024,0.006467555546098285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,2560,0.018427555759747822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,3584,0.020682666036817763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,2048,0.017087111870447796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,512,0.004132444245947732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,1536,0.01612711118327247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,256,0.002955555501911375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,768,0.005305777821275923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,1024,0.01479022204875946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,128,0.002418666664097044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,768,0.014319999350441827
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,64,0.002040000011523565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,512,0.013734222286277346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,32,0.0022088889446523455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,256,0.013020444247457715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,128,0.012663111090660095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,64,0.012494222157531314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,16384,0.018220444520314533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,65536,0.10012977653079563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,32,0.012395555774370829
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,12288,0.01441333360142178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,10240,0.012796444197495779
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,8192,0.007419555551475949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,16384,0.04836977852715386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,65536,0.1546880006790161
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,7168,0.007015110717879401
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,6144,0.010558221903112201
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,12288,0.03997333182228936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,10240,0.035290665096706815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,5120,0.00849866701496972
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,8192,0.030918220678965252
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,7168,0.028720888826582167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,4096,0.00814577771557702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,3584,0.007666666474607255
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,6144,0.02641155487961239
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,3072,0.007094221810499827
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,5120,0.024205333656734888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,2560,0.006395555618736479
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,2048,0.005751111027267244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,4096,0.021560889151361253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,3584,0.020761777957280476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,1536,0.007786666353543599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,3072,0.01946311030122969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,1024,0.005714666512277391
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,2560,0.01828977796766493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,768,0.004831111265553368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,512,0.0038568890757030914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,2048,0.017096888687875535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,256,0.0029288888391521242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,1536,0.016029333074887592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,1024,0.014884443746672736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,128,0.0024195555597543716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,64,0.0023475556323925653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,768,0.014330666926172046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,512,0.013604444762070974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,32,0.0021973333011070886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,128,0.012608889076444836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,256,0.012883555557992725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,64,0.012524444195959302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,16384,0.0075937774446275495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,65536,0.05361422234111362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,32,0.012357333468066322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,12288,0.009226666556464301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,10240,0.008504889077610439
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,8192,0.007442666424645319
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,7168,0.007178666690985362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,16384,0.04838488830460442
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,65536,0.1547093391418457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,12288,0.03998311029540168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,6144,0.006671999891599019
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,8192,0.03067200051413642
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,5120,0.006416000011894438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,7168,0.02867377797762553
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,10240,0.03532088796297709
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,4096,0.0057297779454125305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,3584,0.005642666584915585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,6144,0.02650755478276147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,3072,0.005177777674463061
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,5120,0.02422044508987003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,2560,0.005007110950019625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,4096,0.021553777986102637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,2048,0.00452533322903845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,3584,0.020607999629444547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,1536,0.004121777911980947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,3072,0.01944266590807173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,1024,0.005602666487296422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,768,0.00461422238085005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,2560,0.01830577850341797
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,2048,0.017166222135225933
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,512,0.0037048889531029593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,256,0.0027439999911520216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,1024,0.01477688882086012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,768,0.014323555760913424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,64,0.0021751111166344746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,128,0.0022400000856982335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,512,0.013580444786283704
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,1536,0.015976889265908133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,256,0.012987555729018318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,32,0.0021359999146726397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,128,0.012591111163298288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,16384,0.007883555359310573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,65536,0.024329778220918443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,12288,0.007266666326257918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,10240,0.0070568886068132185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,32,0.012523555921183692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,64,0.012497777740160624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,8192,0.006500444478458828
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,16384,0.048192888498306274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,7168,0.005744888964626525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,10240,0.035375111632876925
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,12288,0.039975110027525164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,65536,0.15430488851335314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,6144,0.005264889034960005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,8192,0.03074666526582506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,5120,0.005120000077618493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,4096,0.004612444589535396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,7168,0.028592000404993694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,3584,0.004886222382386525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,3072,0.004465777840879229
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,6144,0.02643022272321913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,5120,0.023988443944189284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,4096,0.02162133322821723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,3584,0.02056533263789283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,2560,0.004688888788223267
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,2048,0.003999111139112049
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,1536,0.0038577777643998465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,3072,0.019501333435376484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,1024,0.0035777779089079965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,2560,0.01830044388771057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,2048,0.017112000121010672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,512,0.003572444534964032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,768,0.00453333349691497
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,1536,0.01593244406912062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,1024,0.014751111467679342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,256,0.0026622222115596137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,128,0.002234666711754269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,768,0.01422933406300015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,64,0.002159111201763153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,32,0.0021217777911159727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,512,0.013625777430004544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,256,0.013021333350075616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,65536,0.016255999604860943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,128,0.012561777399645912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,16384,0.0069511110583941145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,64,0.012472000386979846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,32,0.012298666768603854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,12288,0.005906666732496685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,10240,0.005973333285914526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,8192,0.005466666486528184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,16384,0.048241777552498706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,7168,0.004888888862397936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,10240,0.035191999541388616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,65536,0.15383999877505833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,6144,0.004761777818202972
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,12288,0.03978133201599121
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,8192,0.03054311209254795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,5120,0.004835555536879433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,4096,0.004265777766704559
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,3584,0.004280000097221798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,7168,0.028347555134031508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,6144,0.026226666238572862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,3072,0.0041644444896115195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,2560,0.004483555754025777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,5120,0.02379111117786831
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,2048,0.0037164443896876443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,4096,0.021516443954573736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,3584,0.020505777663654752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,3072,0.01921422282854716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,1536,0.003681777666012446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,1024,0.003452444449067116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,768,0.003567999849716822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,2560,0.01812533371978336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,512,0.003588444242874781
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,2048,0.01684799955950843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,256,0.0028293333533737394
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,1536,0.015963556038008798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,1024,0.01462222304609087
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,128,0.0022257777551809945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,768,0.014039110806253223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,64,0.002128888956374592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,32,0.0021066665649414062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,512,0.01348977784315745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,256,0.01274311128589842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,65536,0.016050666570663452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,128,0.012430222498046027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,64,0.012433778080675336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,16384,0.006699555450015598
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,32,0.012057777908113269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,12288,0.005679111099905438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,10240,0.005766222046481238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,8192,0.005274666680230035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,7168,0.004675555560323927
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,16384,0.04805955621931288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,12288,0.03950400153795878
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,6144,0.004430222428507275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,10240,0.03499911228815714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,65536,0.15348177485995823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,8192,0.03057510985268487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,5120,0.0043333334227403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,4096,0.004485333131419288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,3584,0.004824888788991504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,7168,0.028171555863486394
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,6144,0.02611911131276025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,3072,0.004072000169091755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,5120,0.023847111397319373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,2560,0.0041368889311949415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,2048,0.0038693332009845306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,4096,0.021292444732454088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,1536,0.0035751110149754416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,3584,0.02034311162100898
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,3072,0.019126221537590027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,1024,0.0034186666210492453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,768,0.00507377791735861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,2560,0.017886221408843994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,2048,0.016861332787407767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,1536,0.015750222735934787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,512,0.00397155558069547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,256,0.0027884443600972495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,1024,0.014517333772447375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,128,0.002234666711754269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,768,0.01387733303838306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,64,0.002135111019015312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,512,0.013343111508422427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,32,0.0020764445265134177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,256,0.012670222255918713
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,128,0.012458666331238218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,64,0.012404444317022959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,32,0.012295111185974546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,10240,1.1903022130330403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,12288,1.4427324930826824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,16384,1.888419469197591
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,8192,1.6798630820380316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,10240,2.0581963857014975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,8192,0.9494400024414062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,12288,2.717782126532661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,7168,0.7810577816433377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,6144,0.6442844602796767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,7168,1.4315919876098633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,5120,0.5327990849812826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,6144,1.2662613126966689
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,16384,3.2739323510064016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,5120,1.0335173077053494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,3584,0.37841778331332737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,4096,0.4604541990492079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,4096,0.8318915367126465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,3584,0.7303226788838705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,2560,0.27995022137959796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,2560,0.5282968944973415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,3072,0.63518312242296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,2048,0.2479893366495768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,3072,0.35490398936801487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,1536,0.3252604537540012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,768,0.16936088932885063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,512,0.11773688263363308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,512,0.09144888983832465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,1024,0.24072800742255318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,2048,0.46686220169067383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,768,0.10899200042088826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,64,0.00959288908375634
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,1536,0.1931582159466214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,1024,0.13535466459062365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,256,0.0878302190038893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,32,0.0070293330483966405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,256,0.058020446035597056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,128,0.08891910976833767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,128,0.020946666598320007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,64,0.08747644556893243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,32,0.09162400166193645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,12288,0.38786133130391437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,10240,0.5123270882500542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,16384,0.5025253295898438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,10240,0.32643644014994305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,12288,0.679384019639757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,16384,0.9083031548394097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,8192,0.42088890075683594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,8192,0.2497004403008355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,7168,0.23485422134399414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,7168,0.3613911204867893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,6144,0.3193039894104004
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,5120,0.26324444346957737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,5120,0.15095999505784777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,6144,0.1987475554148356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,3072,0.15878221723768446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,4096,0.23336354891459146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,3584,0.20450222492218018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,4096,0.12753421730465359
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,3072,0.09725511074066162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,2560,0.130131549305386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,3584,0.1115528874927097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,2048,0.11631555027431911
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,2048,0.06882666879230075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,1536,0.08841688765419854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,1024,0.06009244256549411
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,65536,1.7633546193440754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,2560,0.08353066444396973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,512,0.018311111463440787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,1024,0.03568977779812283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,256,0.005509333478079901
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,768,0.042077332735061646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,1536,0.050564444727367826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,512,0.01649599936273363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,32,0.003216888962520493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,64,0.003698666476541095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,256,0.015166223049163818
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,128,0.014159111513031853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,64,0.014075555735164218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,128,0.004670222186379963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,768,0.018630221486091614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,32,0.014871999621391296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,16384,0.36252888043721515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,12288,0.28125156296624076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,12288,0.515143977271186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,16384,0.6135253376430935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,10240,0.38664711846245664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,10240,0.23141778839959037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,8192,0.31791912184821236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,8192,0.1991413301891751
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,7168,0.27173511187235516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,65536,3.4638392130533853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,7168,0.15363644229041204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,6144,0.2424080106947157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,65536,1.3129759894476998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,4096,0.15654844707912868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,5120,0.19694666067759195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,5120,0.10651466581556533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,6144,0.1278124385409885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,3584,0.13327466117011175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,3584,0.07708888583713107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,2560,0.09646222326490615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,4096,0.09559822082519531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,2048,0.07724622223112318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,2560,0.06362399790022108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,3072,0.12843999597761366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,3072,0.07422488927841187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,2048,0.04911377694871691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,1536,0.03934489025010003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,1024,0.045160889625549316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,1536,0.06588177548514472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,768,0.03273777829276191
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,512,0.0069617778062820435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,1024,0.017467555072572496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,128,0.004376000000370873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,256,0.00508977762526936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,768,0.016543999314308167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,32,0.003118222165438864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,512,0.015089778436554803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,256,0.013925333817799887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,64,0.004090666770935059
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,128,0.013413333230548434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,65536,2.4245084126790366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,64,0.012914666699038612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,32,0.013586666848924426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,16384,0.3083786699506971
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,10240,0.3218631214565701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,16384,0.5095564524332682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,12288,0.4278053442637126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,12288,0.2538879977332221
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,10240,0.19298222329881456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,8192,0.1457377804650201
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,7168,0.2262115478515625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,8192,0.28530044025844997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,7168,0.12396444214714898
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,6144,0.2226088841756185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,4096,0.12827111615075007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,5120,0.1808168888092041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,65536,1.1080488628811307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,4096,0.07458755705091688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,3584,0.13403822316063776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,3072,0.10528444581561619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,5120,0.09924622376759847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,2560,0.0882728894551595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,3072,0.058280891842312284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,6144,0.11856888400183783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,2048,0.07153777943717109
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,1536,0.05425955520735847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,1024,0.04089066717359755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,1024,0.016881777180565726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,1536,0.024907555845048692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,3584,0.07369155353969999
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,2048,0.0462355547481113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,512,0.006707555717892117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,256,0.005061333378156026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,768,0.01610133383009169
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,128,0.00407644444041782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,768,0.01310488912794325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,64,0.0030337776988744736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,256,0.013948443863126965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,512,0.014849777022997538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,32,0.0028817777832349143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,128,0.013056000073750814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,64,0.01277955538696713
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,32,0.013293333351612091
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,2560,0.05532266696294149
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,65536,2.0027955373128257
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,12288,0.19444799423217773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,16384,0.25987201266818577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,12288,0.34600088331434464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,10240,0.15862311257256403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,10240,0.28721777598063153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,16384,0.41277244355943465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,8192,0.12081689304775661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,8192,0.21320888731214735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,7168,0.1843066612879435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,7168,0.10575111044777764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,6144,0.09115288654963176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,4096,0.10437333583831787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,6144,0.17981955740186903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,5120,0.1294728914896647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,65536,0.8825750880771213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,5120,0.07896977663040161
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,3584,0.060228447119394936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,4096,0.07234844234254625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,2560,0.06543555524614122
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,3072,0.08711377779642741
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,3072,0.05413155423270332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,3584,0.10120533572302924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,2048,0.05946933560901218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,2048,0.04028355413013034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,2560,0.05182133449448479
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,1024,0.014939554863505893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,1024,0.02292622294690874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,768,0.00865955568022198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,512,0.0068400001360310455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,256,0.004485333131419288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,768,0.022414222359657288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,128,0.003899555653333664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,512,0.02125599980354309
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,1536,0.04535733328925239
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,1536,0.025892444782786902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,64,0.0028782222006056043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,128,0.019488000207477145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,32,0.0027475555737813315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,64,0.019268444842762418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,32,0.020022221737437777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,256,0.020439111524158053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,65536,1.804720984564887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,16384,0.22611021995544434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,16384,0.36278578970167374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,12288,0.17983110745747885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,10240,0.227896875805325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,12288,0.30205776956346303
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,8192,0.18740088409847686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,10240,0.13115289476182726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,8192,0.11365689171685113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,7168,0.15919466813405356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,65536,0.7756009101867676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,7168,0.09390755494435628
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,6144,0.0801688896285163
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,5120,0.12597689363691542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,4096,0.09093422359890407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,3584,0.0796239972114563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,6144,0.15609688229031032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,4096,0.058824002742767334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,3072,0.06747021940019396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,3584,0.057838221391042076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,2560,0.05682933330535889
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,65536,1.3995929294162328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,2560,0.042640888028674655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,5120,0.07597155703438653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,1024,0.008216888540320927
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,1536,0.03513688842455546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,2048,0.032391998502943255
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,3072,0.048002666897243924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,768,0.006673777682913675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,1536,0.022833777798546687
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,512,0.005519111123349931
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,256,0.004287999951177173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,1024,0.021778666310840186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,768,0.020938666330443487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,128,0.0037155557009908888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,512,0.019867555962668527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,256,0.01881511178281572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,32,0.0025724443710512584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,64,0.0027297778675953546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,128,0.01827022267712487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,32,0.01828177769978841
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,2048,0.05154577891031901
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,64,0.018237334158685472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,16384,0.1927351156870524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,12288,0.2608737680647108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,12288,0.14860978391435412
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,10240,0.11009689172108968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,10240,0.19549332724677193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,16384,0.3473297754923503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,8192,0.15887644555833605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,8192,0.0895964437060886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,6144,0.11893067095014785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,5120,0.09645333555009629
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,7168,0.08020089070002238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,7168,0.15008089277479383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,4096,0.07839644617504544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,3584,0.06797066662046644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,5120,0.06147911151250204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,6144,0.07777777645323011
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,65536,0.7503840128580729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,3072,0.04312888781229655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,2560,0.03310844302177429
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,3584,0.05161688725153605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,4096,0.05244000090493096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,65536,1.1963520050048828
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,2048,0.04448355568779839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,3072,0.06474666463004218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,1536,0.030187557140986126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,1024,0.011757333245542316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,768,0.009463110731707679
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,512,0.005053333524200651
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,1536,0.021511110994550917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,1024,0.020139555136362713
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,768,0.019748444358507793
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,256,0.00397777764333619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,2048,0.02236622240808275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,128,0.0035182221068276297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,512,0.018915555543369718
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,256,0.01773155563407474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,32,0.0025617778301239014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,64,0.0170471105310652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,2560,0.05462933248943753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,128,0.017287999391555786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,32,0.01740622189309862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,64,0.0027022223091787765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,12288,0.11045066515604655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,16384,0.26607823371887207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,10240,0.17685155073801676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,16384,0.16715378231472441
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,12288,0.21492177910274932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,10240,0.09203644593556722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,8192,0.1312453349431356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,8192,0.08202311065461901
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,7168,0.0687111086315579
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,7168,0.1073662241299947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,6144,0.09483466545740764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,5120,0.07793600029415555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,4096,0.06273600127961901
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,5120,0.05323288838068644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,4096,0.04566577739185757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,65536,0.5816035800509983
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,6144,0.0667111078898112
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,2560,0.04046222236421373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,3584,0.055458668205473155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,65536,1.0211466683281791
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,1536,0.009369778136412302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,2048,0.03310577736960517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,3072,0.0483439995182885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,3072,0.03184355629814996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,2560,0.023152000374264185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,2048,0.021993777818149988
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,768,0.009361777868535783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,512,0.0047893333766195504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,3584,0.04161955581771003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,1024,0.01956622302532196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,768,0.01884888940387302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,1024,0.011597333682907952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,256,0.0038577777643998465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,512,0.018128888474570382
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,128,0.0032915555768542816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,256,0.017126222451527912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,1536,0.020890666378868952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,64,0.002599999929467837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,32,0.002452444492114915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,128,0.016788444585270353
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,64,0.016971555021074083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,32,0.017122666041056316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,16384,0.11688621838887532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,12288,0.09018755621380276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,12288,0.16370844841003418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,10240,0.1304088830947876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,16384,0.21687644057803682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,8192,0.1053582231203715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,10240,0.07624444696638319
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,7168,0.08466755681567723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,6144,0.07479466994603474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,5120,0.060772445466783315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,8192,0.06376088990105523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,7168,0.0580257773399353
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,65536,0.5281768904791938
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,5120,0.045455998844570585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,3584,0.043599999613232084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,4096,0.03672800130314297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,65536,0.8151573075188531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,3072,0.03803733322355483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,6144,0.05192177825503879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,2560,0.031833777825037636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,4096,0.049588445160124034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,2560,0.021160889003011916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,2048,0.016532444291644625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,3072,0.02237333357334137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,3584,0.02587822245226966
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,1536,0.012126222252845764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,1024,0.008722667064931657
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,768,0.007357333269384172
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,2048,0.02018222212791443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,512,0.005598222215970357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,1536,0.018952000472280715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,256,0.0041546668443414895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,128,0.003176888864901331
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,768,0.017557332913080852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,64,0.002489777695801523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,1024,0.017862222260899015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,256,0.015640000502268474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,64,0.01501511037349701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,32,0.0023253332409593794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,512,0.016650666793187458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,128,0.015338665909237333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,32,0.015432000160217285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,16384,0.18962132930755615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,12288,0.1363475587632921
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,16384,0.10359466738171047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,10240,0.06844355662663777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,12288,0.08069511254628499
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,10240,0.1272488832473755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,7168,0.07501333289676242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,8192,0.09248977899551392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,8192,0.05753866831461588
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,65536,0.42800267537434894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,7168,0.05221422182189094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,5120,0.05401600069469876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,6144,0.07480177614423963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,4096,0.044490665197372437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,3584,0.03883466786808438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,65536,0.7186791102091471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,4096,0.023519999451107446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,6144,0.046929776668548584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,3072,0.03412977854410807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,5120,0.04103466537263658
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,2048,0.01460977726512485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,3072,0.021520000365045335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,3584,0.02269688910908169
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,1536,0.011919111013412476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,2560,0.020427554845809937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,1024,0.00850044439236323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,2560,0.030023998684353296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,768,0.007180444068378872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,1536,0.018200000127156574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,512,0.005523555394675996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,256,0.004129777765936322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,128,0.0033404444240861465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,2048,0.019498666127522785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,512,0.01594577729701996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,768,0.016516443755891588
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,64,0.0024195555597543716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,256,0.014921777778201632
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,128,0.014543111125628153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,32,0.002283555558986134
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,1024,0.01723288827472263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,64,0.014462222655614218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,32,0.014709333578745524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,16384,0.15993777910868326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,12288,0.07234578000174628
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,12288,0.13112266858418783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,16384,0.09189511007732815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,10240,0.06102399693595039
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,8192,0.07984889215893216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,65536,0.37905510266621906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,7168,0.06566755639182197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,10240,0.09873600138558282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,8192,0.0517155561182234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,6144,0.0582266648610433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,5120,0.04761955473158094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,65536,0.6214959886338975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,4096,0.03944177760018243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,7168,0.04714044597413805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,5120,0.029073778125974867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,3584,0.03416533271471659
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,3584,0.021831999222437542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,4096,0.022805333137512207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,2048,0.014371555712487964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,6144,0.04567377765973409
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,3072,0.02072266737620036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,2560,0.019783111082182992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,3072,0.028386665715111628
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,1024,0.008325333396593729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,1536,0.01752000053723653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,768,0.006918222126033571
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,2560,0.018567999203999836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,512,0.005344888816277186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,1536,0.011695110963450538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,256,0.003988444391224119
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,2048,0.018611555298169453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,768,0.015817777978049386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,1024,0.016435555285877652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,512,0.015426667200194465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,128,0.003160888950030009
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,64,0.0023502223193645477
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,32,0.0020657777786254883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,256,0.01424711114830441
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,128,0.013927999469969006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,64,0.013800000150998434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,32,0.014103111293580798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,16384,0.12966222233242458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,12288,0.0982195536295573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,16384,0.07952000035179986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,10240,0.08331644535064697
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,12288,0.06334933307435778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,8192,0.06722311178843181
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,8192,0.0454053348965115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,65536,0.32776000764634877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,7168,0.056664003266228564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,6144,0.05046311020851135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,7168,0.03984266519546509
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,10240,0.053295112318462796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,65536,0.5179333157009548
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,4096,0.0337759984864129
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,3072,0.020581333173645865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,3584,0.028768890433841284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,4096,0.022521777285469904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,6144,0.028534223635991413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,3584,0.02123733361562093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,5120,0.04169333312246534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,1536,0.01147822207874722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,5120,0.024699555502997503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,2048,0.01407911048995124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,2048,0.018056000272432964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,2560,0.017431111799346078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,2560,0.01941066649225023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,1024,0.008107555409272512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,3072,0.02036888897418976
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,768,0.006670222100284364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,1536,0.016976000534163583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,256,0.0037955554823080697
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,1024,0.01608888970481025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,768,0.015465777781274585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,128,0.002987555538614591
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,512,0.01496088835928175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,64,0.0022479999396536085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,512,0.005168000029193031
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,32,0.0020373333245515823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,256,0.013741333451535968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,128,0.013491555220550962
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,64,0.013419555293189155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,32,0.013614222407341003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,16384,0.10422933101654053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,12288,0.08010488748550415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,16384,0.06760711140102811
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,10240,0.06721066766315036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,12288,0.05413688884841072
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,8192,0.05483377642101712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,7168,0.0462408893638187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,65536,0.4211004575093587
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,6144,0.0409457782904307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,7168,0.02836266822285122
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,8192,0.03349688980314467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,10240,0.04571555720435249
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,65536,0.2750195662180583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,4096,0.02341777748531765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,3584,0.01870044403605991
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,6144,0.026226666238572862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,3072,0.016359110673268635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,5120,0.024041778511471216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,4096,0.02160444524553087
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,5120,0.03902666767438253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,3584,0.020654223031467862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,2048,0.011419555379284753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,2560,0.018590221802393597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,1536,0.009287999735938178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,3072,0.01958044370015462
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,2560,0.013927110367351107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,2048,0.01751377847459581
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,768,0.005642666584915585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,512,0.004386666748258802
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,1024,0.015207999282413058
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,256,0.0032640000184377036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,768,0.01460800071557363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,1536,0.016427555018001132
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,128,0.0027786667148272195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,64,0.0022008888837363985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,512,0.014166222678290473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,256,0.013249778085284762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,32,0.001981333312061098
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,64,0.012844444149070315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,1024,0.006803555621041193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,128,0.01292711082432005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,32,0.0129466669427024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,16384,0.08008622460895114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,12288,0.06159466505050659
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,10240,0.051685333251953125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,16384,0.05728089147143894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,10240,0.034456888834635414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,12288,0.04549955659442478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,65536,0.3195893234676785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,65536,0.21683200200398764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,7168,0.041999111572901406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,8192,0.030252443419562444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,6144,0.03596622082922194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,7168,0.027963555521435205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,6144,0.025965332984924316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,5120,0.027943111128277246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,3584,0.018411555223994784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,8192,0.047275556458367236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,5120,0.02360444433159298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,4096,0.020610666937298246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,3072,0.016061334146393668
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,2560,0.013655111193656921
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,4096,0.021378666162490845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,2048,0.01129066695769628
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,3072,0.019280888968043856
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,1536,0.009069333473841349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,1024,0.006616888774765863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,3584,0.020390222469965618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,2560,0.018276444739765592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,1536,0.016171556380059984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,768,0.005429333282841577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,1024,0.01481066644191742
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,768,0.014548444085650973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,512,0.00425155543618732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,2048,0.01718577742576599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,256,0.0031439999325407874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,128,0.00258933338854048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,64,0.0021191111041439907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,32,0.0020248889923095703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,512,0.013817778064144982
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,128,0.012669333153300814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,64,0.012633778154850006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,32,0.012817777693271637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,256,0.012991999586423239
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,16384,0.054584887292650014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,65536,0.22326845592922637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,10240,0.03588533401489258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,12288,0.042287998729281955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,8192,0.01738222274515364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,16384,0.04736977815628052
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,65536,0.1596471071243286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,7168,0.015384889311260648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,12288,0.03865777783923679
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,10240,0.03436533278889126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,8192,0.030003554291195337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,6144,0.013755555782053204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,4096,0.010265777508417765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,5120,0.012294222083356647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,7168,0.02784622377819485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,3584,0.009764444496896531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,5120,0.023489778240521748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,4096,0.021213332811991375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,2560,0.009407111340098912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,3072,0.008914666871229807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,6144,0.025881777207056682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,2048,0.00811288919713762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,3072,0.019189332922299702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,1536,0.008845333423879411
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,1024,0.005945777727497949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,2560,0.018199111024538677
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,768,0.005325333525737126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,1536,0.01607377827167511
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,512,0.004156444635656145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,3584,0.020346666375796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,1024,0.014709333578745524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,2048,0.017066667477289837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,256,0.003006222140457895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,128,0.002424000038041009
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,768,0.014349333114094205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,64,0.002005333287848367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,32,0.002185777864522404
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,512,0.013822221921549903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,256,0.013020444247457715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,128,0.012597333225939008
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,64,0.012528888881206512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,32,0.012713777522246042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,12288,0.03288177649180094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,16384,0.04242577817704943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,10240,0.01707555519209968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,65536,0.16391021675533718
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,8192,0.013804444836245643
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,12288,0.03853244582811991
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,16384,0.04679822259479099
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,65536,0.14823910925123426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,10240,0.03400977783732944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,6144,0.011979555918110741
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,8192,0.029789331886503432
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,5120,0.007050666544172499
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,7168,0.013344888885815939
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,4096,0.008948444492287105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,7168,0.02788355615403917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,3584,0.00872444444232517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,5120,0.023334221707450017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,2560,0.007295110987292395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,6144,0.025695110360781353
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,4096,0.020959110723601446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,3584,0.020097777247428894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,1536,0.008759111166000366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,2048,0.006618666566080517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,1024,0.005821333163314396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,2560,0.018016889691352844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,3072,0.019157333506478202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,2048,0.01682933337158627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,1536,0.01589777734544542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,768,0.004880889008442561
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,3072,0.008010666403505538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,512,0.00407644444041782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,128,0.002377777877781126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,256,0.0029484445436133277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,1024,0.01462577780087789
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,768,0.01401600076092614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,64,0.0019697778754764134
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,32,0.002177777803606457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,256,0.012914666699038612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,512,0.01351377781894472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,128,0.012692444854312472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,16384,0.018401778406567044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,64,0.012574222352769641
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,32,0.012605333493815528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,12288,0.013479111095269522
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,65536,0.10516888565487331
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,10240,0.012054222325483957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,16384,0.04668622215588888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,8192,0.0069573331210348345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,7168,0.006677333265542984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,12288,0.038373334540261164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,6144,0.010438222024175854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,10240,0.03399822115898132
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,65536,0.14793866210513645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,5120,0.00925422211488088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,7168,0.02769688930776384
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,4096,0.008023999631404877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,3584,0.0074888889988263445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,8192,0.02979555394914415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,6144,0.025654221574465435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,3072,0.006870222174459034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,4096,0.020942222740915086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,2560,0.006271111054552927
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,5120,0.023390221926901076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,2048,0.005658666706747479
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,1536,0.007638221813572778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,3584,0.020053333706325956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,1024,0.005679111099905438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,3072,0.01904355486234029
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,2560,0.018014222383499146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,768,0.004785777793990241
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,512,0.0037768888804647657
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,2048,0.016727111405796476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,1536,0.015912888778580558
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,256,0.0028657778683635923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,128,0.0022960000981887183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,1024,0.01461955573823717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,64,0.002209777840309673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,768,0.014127110441525778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,32,0.0021359999146726397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,256,0.012851555314328937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,128,0.012633778154850006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,512,0.013452444639470843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,16384,0.007254222200976477
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,64,0.012462221913867526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,65536,0.05340888765123156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,32,0.01279022213485506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,12288,0.006454222318198945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,10240,0.00812533332241906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,8192,0.007161777880456712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,16384,0.046801779005262584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,12288,0.03835466504096985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,7168,0.0075039996041191956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,6144,0.00685422205262714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,8192,0.02980533242225647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,65536,0.14791556199391684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,5120,0.006275555739800136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,10240,0.034095999267366194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,7168,0.027765333652496338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,4096,0.005611555443869696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,3584,0.005535111245181825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,6144,0.025607999828126695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,5120,0.023395554886923894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,2560,0.004946666873163647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,3072,0.005097777893145879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,2048,0.004471999903519948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,1536,0.0041146667467223275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,3584,0.020095111595259774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,3072,0.01901511185699039
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,4096,0.021026665965716045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,2560,0.01798488861984677
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,2048,0.016694222887357075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,1024,0.005547555370463266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,768,0.004569777597983678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,512,0.0036524443162812125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,256,0.0027537778433826235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,128,0.0022497777309682635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,1024,0.014626666903495789
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,1536,0.01588622232278188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,768,0.01407999959256914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,64,0.0021600000974204806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,512,0.013415111435784234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,32,0.0021395554973019496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,256,0.012813333008024426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,65536,0.021469333105617102
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,16384,0.00758133331934611
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,64,0.012497777740160624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,32,0.012668444050682915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,12288,0.0070675553547011475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,10240,0.0068915556702348925
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,8192,0.005909333212508096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,128,0.012535110943847232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,16384,0.046739555067486234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,12288,0.03853866789076064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,7168,0.005642666584915585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,65536,0.1472053395377265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,6144,0.005159111072619756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,10240,0.034105777740478516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,5120,0.005054222212897407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,4096,0.004534222185611725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,8192,0.029779556724760268
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,3584,0.004811555561092165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,7168,0.027685332629415724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,6144,0.025677333275477093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,3072,0.004435555388530095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,4096,0.021000888612535264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,2560,0.004664888812435998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,5120,0.023150222169028387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,2048,0.003953777667548922
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,3584,0.020042666130595736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,3072,0.018921777606010437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,1536,0.003999999827808804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,1024,0.003620444486538569
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,2560,0.01809511085351308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,768,0.00453333349691497
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,512,0.003591999825504091
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,2048,0.016795555750528973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,1536,0.015665777855449252
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,1024,0.01460622251033783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,256,0.002834666727317704
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,128,0.0022248888595236675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,768,0.014064888159434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,64,0.002151111140847206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,512,0.01333333303531011
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,32,0.0021342221233579847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,256,0.012860444684823355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,65536,0.016209777858522203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,128,0.01262399968173769
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,16384,0.006879111131032308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,64,0.012297777665985955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,12288,0.005803555664088991
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,10240,0.0058986664646201665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,32,0.01290844463639789
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,8192,0.005423111220200856
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,16384,0.04657333427005344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,12288,0.03826755616399977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,10240,0.033965332640541926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,65536,0.14730132950676814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,7168,0.004844444493452708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,6144,0.0047048889100551605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,8192,0.02970577942000495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,5120,0.00479911102188958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,4096,0.004224888980388641
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,7168,0.027457776996824477
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,3584,0.004886222382386525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,5120,0.023065777288542852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,6144,0.025480000509156123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,3072,0.004143110993835661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,2560,0.004448889030350579
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,4096,0.02088000045882331
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,2048,0.0037164443896876443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,3584,0.019888000355826486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,1536,0.0036862223512596558
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,1024,0.0034337778472238113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,3072,0.018848000301255118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,768,0.003568888952334722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,2560,0.017735110388861764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,2048,0.01657777859105004
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,512,0.00397155558069547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,1536,0.015486222174432544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,256,0.0027964444210131965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,128,0.0022479999396536085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,1024,0.01445777714252472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,768,0.013933334085676404
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,64,0.0021199999998013177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,512,0.013431111143694984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,32,0.0021066665649414062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,256,0.012621333201726278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,65536,0.015903110305468243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,128,0.012247111234400006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,16384,0.006680000159475539
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,12288,0.005682666682534748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,64,0.012297777665985955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,32,0.01254755589697096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,10240,0.005748444547255834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,8192,0.0052800000541739995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,16384,0.04643733302752177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,7168,0.00462666650613149
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,12288,0.03802666730350918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,10240,0.03369066781467862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,6144,0.004420444369316101
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,8192,0.029547555579079524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,5120,0.004280000097221798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,65536,0.14720266395144993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,4096,0.0045422220395671
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,7168,0.02751644452412923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,3584,0.004452444612979889
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,6144,0.02533066603872511
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,3072,0.004233777936961916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,5120,0.022859555151727464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,2560,0.0041422223051389056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,2048,0.003908444609906939
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,4096,0.020842666427294414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,3584,0.019817777805858187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,1536,0.003585777762863371
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,3072,0.018587556150224473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,1024,0.006180444525347815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,768,0.005051555732885997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,2560,0.01773955590195126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,512,0.0038986665507157645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,2048,0.01645511057641771
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,256,0.0027919999427265594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,1536,0.015465777781274585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,1024,0.014402666025691561
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,128,0.0021768889079491296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,64,0.0021111110432280437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,768,0.013611555927329592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,512,0.013175110850069257
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,32,0.0020764445265134177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,256,0.012778666284349231
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,128,0.012303111453851065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,64,0.01240533341964086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,32,0.012342222034931183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,10240,1.1797413296169705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,12288,1.4254141913519964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,16384,1.8597545623779297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,10240,2.0478248596191406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,8192,0.9127546946207682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,8192,1.671529769897461
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,12288,2.7061483595106335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,7168,0.8325004577636719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,7168,1.4256106482611761
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,5120,0.5210800170898438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,6144,0.6392515500386556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,16384,3.2635947333441844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,4096,0.41628265380859375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,5120,1.0257707171969943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,6144,1.2592862447102864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,3072,0.3222257826063368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,3584,0.3967742125193278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,3072,0.6310853428310818
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,3584,0.7247093518575033
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,4096,0.9162986543443469
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,2560,0.27269066704644096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,2048,0.421561770968967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,2048,0.22543732325236002
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,2560,0.522832022772895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,1024,0.21846489111582437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,1024,0.13143022855122885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,768,0.09741689099205865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,128,0.01758311192194621
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,1536,0.35072888268364805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,768,0.18018310599856904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,1536,0.18815378348032633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,64,0.006040888941950268
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,512,0.117923551135593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,32,0.00442755553457472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,256,0.08133688900205824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,512,0.08497955401738484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,64,0.0846995578871833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,128,0.0853946672545539
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,256,0.05885510974460178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,32,0.08313510815302531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,12288,0.3838062286376953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,16384,0.4913013246324327
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,10240,0.3198435571458605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,12288,0.7221920225355359
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,10240,0.5642417801751031
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,16384,0.8082568910386828
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,8192,0.2597057819366455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,6144,0.18376533190409342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,7168,0.35919822586907285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,8192,0.4590640068054199
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,7168,0.22945777575174967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,6144,0.34987200631035703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,5120,0.25856444570753306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,5120,0.1473031176461114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,4096,0.2089022265540229
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,4096,0.11385156048668756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,3584,0.17891377872890898
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,3072,0.15213244491153294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,65536,1.6809315151638453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,2048,0.10143555535210504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,2560,0.0745146671930949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,3072,0.09453777472178142
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,2048,0.061722669336530894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,1536,0.049489776293436684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,3584,0.10875466797086929
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,2560,0.1407359970940484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,1024,0.033835556772020124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,1024,0.05202133456865946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,512,0.01480977733929952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,1536,0.08551377720303005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,256,0.005410666680998272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,512,0.01569688816865285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,768,0.04430311255984836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,128,0.004873777843183941
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,256,0.01421333352724711
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,64,0.003083555648724238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,128,0.01346577786737018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,32,0.0029288888391521242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,768,0.016493333710564505
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,64,0.013311111264758639
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,32,0.013285333083735572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,16384,0.3496195475260417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,12288,0.5116088655259874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,12288,0.2704711225297716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,10240,0.42499645551045734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,10240,0.22729778289794922
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,16384,0.6839653121100532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,8192,0.29794489012824166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,8192,0.18088444073994955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,65536,3.178347481621636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,7168,0.15223199791378444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,5120,0.193686220380995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,7168,0.2964435683356391
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,6144,0.2382133271959093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,5120,0.10527466403113471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,65536,1.279949294196235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,4096,0.15466754966311985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,6144,0.13962754938337538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,3072,0.11552266279856364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,3584,0.1331848833296034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,3072,0.06699022319581774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,2560,0.05785422192679512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,2560,0.09698844618267483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,3584,0.0843671096695794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,4096,0.09444799688127305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,1536,0.0584755539894104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,1536,0.03900444507598877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,768,0.030001779397328694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,1024,0.044414222240448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,512,0.006996444529957241
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,2048,0.05214488837454054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,2048,0.08736711078219944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,1024,0.015488889482286243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,256,0.004523555437723796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,768,0.014719999498791166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,128,0.003576888806290097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,64,0.0028400001012616684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,512,0.013980444934633044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,256,0.01295466638273663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,32,0.002699555622206794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,128,0.01220533334546619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,32,0.012210666305489011
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,64,0.012402666939629449
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,65536,2.400879118177626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,16384,0.2949315441979302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,12288,0.4244506624009874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,16384,0.5683449109395345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,10240,0.3199208842383491
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,10240,0.1863084501690335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,12288,0.24177866511874727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,8192,0.28410667843288845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,8192,0.1553235583835178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,6144,0.21536355548434785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,6144,0.10424977540969849
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,7168,0.12671644157833523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,7168,0.26663822597927517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,65536,1.0686364703708224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,5120,0.19112888971964517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,4096,0.07319200038909912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,3584,0.06513955858018664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,4096,0.15313511424594456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,5120,0.09324889050589667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,3584,0.13402044773101807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,3072,0.10891377925872803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,2560,0.08096800247828166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,2048,0.06570844517813788
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,2560,0.04959377646446228
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,1536,0.0551528897550371
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,3072,0.057586669921875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,2048,0.04548177785343594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,1024,0.03340889016787211
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,1536,0.024506666594081458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,512,0.006315555423498154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,768,0.014222222897741528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,768,0.008858666651778752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,256,0.004295999805132548
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,128,0.0031822222388452957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,64,0.0027751111321979095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,512,0.013423999978436364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,32,0.00263288886182838
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,128,0.012058667010731168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,1024,0.01537333263291253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,256,0.012784889174832238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,64,0.01202488856183158
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,32,0.011906666888131036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,65536,2.175128936767578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,12288,0.18687200546264648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,12288,0.3423022164238824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,16384,0.26954399214850533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,10240,0.256601783964369
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,16384,0.4098462263743083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,10240,0.1664888858795166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,8192,0.11638222800360785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,6144,0.14684000280168322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,7168,0.10261066754659016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,7168,0.18625777297549775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,8192,0.21210755242241752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,5120,0.11831555101606582
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,5120,0.07602755228678386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,3584,0.08345511224534775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,6144,0.08816978004243638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,4096,0.06952177815967135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,65536,0.9042764239841037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,2560,0.06053688791063097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,3072,0.052003555827670626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,3072,0.08081510994169447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,2048,0.04901422063509623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,2560,0.04968711071544223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,4096,0.10707200235790676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,1024,0.014841778410805596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,1536,0.037316444847318865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,2048,0.039055999782350324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,1536,0.0229111115137736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,768,0.0075626663035816616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,3584,0.06296977731916639
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,256,0.004543999830881755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,128,0.0034204444123639
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,768,0.020465777979956735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,1024,0.021038222644064162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,32,0.0025840000145965153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,64,0.003175111073586676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,256,0.018992889258596633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,512,0.005799110978841782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,128,0.01853155593077342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,512,0.02001066671477424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,64,0.018421333697107103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,65536,1.5958062277899847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,32,0.01829866733815935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,16384,0.2187573379940457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,12288,0.300471994611952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,16384,0.3606044451395671
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,10240,0.22548354996575248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,12288,0.1741573280758328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,8192,0.18584711021847197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,10240,0.12809067302280003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,8192,0.10024888647927178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,7168,0.14602577686309814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,7168,0.09688622421688503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,6144,0.12843110826280382
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,65536,0.7497608926561145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,5120,0.06670755810207792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,5120,0.10545421971215142
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,6144,0.07780800263086955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,3584,0.07334666781955294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,3072,0.06319377819697063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,2560,0.05342310998174879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,3072,0.046206222640143506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,2560,0.04110577702522278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,4096,0.06135288874308268
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,4096,0.09434133105807835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,1536,0.032999111546410456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,1024,0.009014222357008193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,1536,0.020776000287797716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,2048,0.02880711025661892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,3584,0.0556817783249749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,1024,0.019522666931152344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,2048,0.048511111074023776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,768,0.0074240002367231585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,512,0.005751111027267244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,256,0.00443822228246265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,128,0.0034862222770849862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,64,0.003079111170437601
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,512,0.018619555566045973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,768,0.019121777680185106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,32,0.002493333278430833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,128,0.01742577718363868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,256,0.018070222602950204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,65536,1.505125363667806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,64,0.01715022159947289
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,32,0.017147555947303772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,16384,0.30950045585632324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,12288,0.1433768934673733
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,10240,0.1921235587861803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,16384,0.18579110834333631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,12288,0.2880568769243029
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,7168,0.12443555725945367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,8192,0.08618666728337605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,8192,0.17536444134182402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,10240,0.10775822401046753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,7168,0.08511377705468072
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,65536,0.657592879401313
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,6144,0.1105697751045227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,4096,0.08121689160664876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,5120,0.10101511081059773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,4096,0.050145778391096324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,5120,0.05949066744910347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,6144,0.07107644610934787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,3584,0.045920888582865395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,3584,0.07095999850167169
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,3072,0.061436441209581166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,65536,1.1829342312282987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,2560,0.0517759985393948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,2560,0.031097776359981958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,2048,0.02000711030430264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,3072,0.04442310995525784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,1024,0.008904000123341879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,1536,0.028601778878106013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,768,0.007302222152551015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,256,0.00424177779091729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,512,0.005533333453867171
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,1024,0.018491556247075398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,2048,0.040503998597462974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,768,0.01753777762254079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,1536,0.01923111081123352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,256,0.016618667377365958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,64,0.0028862222615215513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,32,0.002452444492114915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,128,0.0031724443866146933
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,64,0.016591111818949383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,32,0.016037333342764113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,128,0.016300444801648457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,512,0.017315554949972365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,10240,0.15854756037394205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,12288,0.21313599745432535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,12288,0.10627822081247966
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,16384,0.14664355913798013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,16384,0.25681689050462514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,8192,0.1298311154047648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,7168,0.10458399852116902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,6144,0.09267199701733059
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,10240,0.09801689121458267
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,7168,0.06619733572006226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,65536,0.5529164208306206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,5120,0.07597333192825317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,8192,0.07311377922693889
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,6144,0.05840088923772176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,3584,0.05321777860323588
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,5120,0.05579644441604614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,4096,0.061174220508999295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,4096,0.04363911019431221
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,3072,0.04635999931229485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,3584,0.03953333364592658
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,2560,0.04389333393838671
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,2048,0.03163999981350369
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,1536,0.013386666774749756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,65536,1.0150044759114583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,1024,0.009335999687512716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,2560,0.020651555723614164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,2048,0.019550222489568923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,512,0.005647111270162795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,1536,0.018768888380792405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,256,0.004086222085687849
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,1024,0.017663111289342243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,768,0.017247110605239868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,768,0.007631999750932057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,512,0.016751110553741455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,3072,0.029261334074868098
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,64,0.002781333401799202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,256,0.016237333416938782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,128,0.0031511110977994073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,128,0.01606755620903439
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,32,0.0024337776833110386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,32,0.015903110305468243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,64,0.01603200038274129
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,16384,0.19787822829352486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,12288,0.15017599529690212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,16384,0.11153688695695664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,12288,0.08647822009192573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,10240,0.1346773306528727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,10240,0.07328444719314575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,8192,0.09649778074688381
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,7168,0.08398044771618313
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,8192,0.060886224110921226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,5120,0.06028978029886881
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,6144,0.07505689064661662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,65536,0.4973164664374457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,4096,0.04929866724544101
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,5120,0.04326755470699734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,7168,0.06051199966006809
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,4096,0.032608889871173434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,65536,0.7517191039191352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,3584,0.04793511165512932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,6144,0.049230221245023936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,2560,0.03158400124973721
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,3072,0.03780622283617655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,3072,0.018965333700180054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,2048,0.015437333120240105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,1536,0.010739555789364709
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,1024,0.007058666812049017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,768,0.006165333506133821
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,1536,0.016746666696336534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,2048,0.017270222306251526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,512,0.004609777695602841
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,256,0.003934222377008862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,3584,0.022341334157519873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,768,0.015303111738628812
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,2560,0.018311111463440787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,128,0.0028364445186323593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,64,0.0031742221779293488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,1024,0.015797333584891427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,512,0.0151555554734336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,256,0.014185777968830533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,128,0.014125333891974555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,32,0.0026177778426143858
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,64,0.01404888927936554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,32,0.013883555928866068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,16384,0.17375289069281685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,12288,0.13055377536349827
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,16384,0.09962577952278985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,10240,0.10544800096087986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,12288,0.0852888888782925
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,8192,0.08520711130566067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,10240,0.06562399864196777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,7168,0.07417600022421943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,5120,0.05334755447175768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,6144,0.06546311246024238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,7168,0.049716442823410034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,6144,0.0442791117562188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,3584,0.03795555565092299
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,4096,0.04369599951638115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,8192,0.059902224275800914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,65536,0.4038862122429742
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,3584,0.018939556346999276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,65536,0.6599955558776855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,3072,0.03341333402527703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,2560,0.026389333936903212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,5120,0.03851200143496195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,2048,0.011494222614500256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,1024,0.0070186663005087115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,1536,0.009905777871608734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,2560,0.01770755483044518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,2048,0.016498666670587327
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,1536,0.015936889582210116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,1024,0.014877332581414117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,768,0.0057946667075157166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,512,0.004414222306675381
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,4096,0.021310221817758348
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,256,0.003656889001528422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,3072,0.0181057784292433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,64,0.0025946667624844448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,512,0.014293332894643148
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,32,0.002521777732504739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,256,0.013635555903116861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,128,0.013384888569513956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,768,0.014913777510325113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,64,0.013352000051074557
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,128,0.003157333367400699
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,32,0.013246222502655454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,12288,0.11193511221143936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,16384,0.14903377162085638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,10240,0.09107822179794312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,16384,0.08723911311891343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,12288,0.07583644655015734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,8192,0.07359200053744845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,7168,0.06433600187301636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,8192,0.04881511131922404
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,65536,0.35421154234144425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,7168,0.04441155658827888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,5120,0.046932445632086865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,6144,0.05757511324352688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,65536,0.5678924454583062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,4096,0.0380675560898251
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,3072,0.029380444023344252
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,5120,0.024715556038750544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,4096,0.01907822158601549
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,3584,0.033321777979532875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,6144,0.0397093329164717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,10240,0.05832088655895657
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,2560,0.015603555573357476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,2048,0.011375111010339526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,1536,0.010021333065297868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,1024,0.006853333363930385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,3072,0.017383999294704862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,2560,0.016872000363137987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,2048,0.01588622232278188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,3584,0.01811644434928894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,512,0.004678222040335338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,1024,0.014310222533014087
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,256,0.0035111109415690103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,768,0.013954665925767688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,768,0.005643555687533484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,128,0.0026577777332729767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,1536,0.015091554986106025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,512,0.01368711143732071
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,64,0.0025271111064487034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,256,0.01312711089849472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,32,0.0020737778395414352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,64,0.012772444221708508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,128,0.012828444441159567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,32,0.012826667063766055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,16384,0.1195404397116767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,16384,0.0751475559340583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,10240,0.07649155457814534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,12288,0.09042488866382176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,12288,0.06014044417275322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,8192,0.06306133005354139
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,65536,0.30367199579874676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,10240,0.05055377880732218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,7168,0.05452355411317614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,65536,0.4748729070027669
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,7168,0.03875911235809326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,5120,0.040758222341537476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,6144,0.04929866724544101
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,8192,0.04297777679231432
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,4096,0.03269244564904107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,3584,0.02646222213904063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,4096,0.018219555417696636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,3072,0.017497777938842773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,6144,0.024337778488794964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,3584,0.017841777867741056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,5120,0.019878221882714164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,1536,0.009927999642160203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,3072,0.017127111554145813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,2048,0.01130222198035982
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,2560,0.016227554943826463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,1024,0.006670222100284364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,768,0.006001777946949005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,2048,0.015456888410780164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,2560,0.01384800010257297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,1536,0.014578666951921252
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,256,0.0032711111836963226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,512,0.004671111288997862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,128,0.002667555585503578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,768,0.013528000149461957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,64,0.0022088889446523455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,1024,0.014167111780908374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,256,0.012611555556456247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,32,0.001997333433892992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,128,0.012539555629094442
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,512,0.013271110753218332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,64,0.012331555287043253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,32,0.012327999704413943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,16384,0.09593422545327081
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,16384,0.06334133280648126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,12288,0.07317155599594116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,10240,0.061794665124681264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,8192,0.0501226650344001
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,10240,0.04306755463282267
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,7168,0.04426044556829664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,8192,0.03161688976817661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,12288,0.05539822247293261
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,65536,0.2533671061197917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,6144,0.03931644558906555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,65536,0.3840160104963515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,5120,0.032855110036002264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,4096,0.011639110743999481
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,7168,0.022515555222829182
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,3584,0.010636444720957013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,4096,0.017624000708262127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,3072,0.009568000005351173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,2560,0.008790222307046255
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,3584,0.017214222086800467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,6144,0.020735111501481798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,3072,0.016414221790101793
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,2048,0.008421333299742805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,5120,0.019377777973810833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,1536,0.009000889129108852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,1024,0.006585777633719974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,2560,0.015614221493403116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,768,0.005361777626805835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,512,0.0041982221106688184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,1536,0.014058666096793281
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,256,0.0031511110977994073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,768,0.013005333642164866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,512,0.012521777715947894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,128,0.0026382222357723448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,2048,0.014900444282425774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,256,0.01202755586968528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,64,0.002130666747689247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,1024,0.013404444687896304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,32,0.0019431112127171622
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,128,0.011948444777064852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,64,0.011737777955002256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,32,0.011873777541849347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,16384,0.07329866621229383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,12288,0.05824089050292969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,16384,0.05280799998177422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,10240,0.047431998782687716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,12288,0.042360001140170626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,65536,0.19753600491417778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,8192,0.03870844509866502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,65536,0.30203734503852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,10240,0.02935289012061225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,6144,0.03000266684426202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,5120,0.011117333339320289
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,7168,0.03529422150717841
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,7168,0.022106667359670002
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,4096,0.009665778113736046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,8192,0.023342221975326538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,3584,0.0092604441775216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,3072,0.008413333031866286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,6144,0.020603555772039626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,2560,0.009500444763236577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,5120,0.019211555520693462
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,3584,0.016940444707870483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,4096,0.017582222819328308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,1536,0.008868444297048781
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,3072,0.016215999921162922
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,2048,0.014722666806644864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,1024,0.006527110934257507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,2048,0.006931555353932911
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,768,0.005320000151793162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,2560,0.015592889653311836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,1536,0.013898667362001208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,512,0.004146666576464971
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,1024,0.01330933305952284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,256,0.0030666666312350165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,128,0.0023911111056804657
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,64,0.0020826665891541377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,32,0.0022435554613669715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,768,0.012919999659061432
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,512,0.012470222181744046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,256,0.011967110964987012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,128,0.011671110987663269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,64,0.011609777808189392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,32,0.011364444262451597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,16384,0.04999022351370918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,12288,0.038615110847685076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,16384,0.04024622175428603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,10240,0.032893333170149065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,65536,0.20156444443596733
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,65536,0.13399910926818848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,8192,0.01276000009642707
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,10240,0.026554667287402686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,12288,0.029476443926493328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,7168,0.012038222617573209
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,6144,0.010715555813577441
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,8192,0.023399111297395494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,5120,0.009502222140630087
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,4096,0.008211555580298105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,3584,0.007717333734035492
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,7168,0.02199111051029629
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,6144,0.020543111695183646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,5120,0.01904444396495819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,3072,0.007063111497296228
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,3584,0.016925333274735343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,2560,0.006543111056089401
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,2048,0.0057813334796163775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,4096,0.017652443713612027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,1536,0.007855999800893996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,3072,0.016075554821226332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,1024,0.005882666756709416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,768,0.00490399988161193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,2048,0.014615999327765571
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,2560,0.015480000111791821
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,1536,0.013996443814701505
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,256,0.0029928889125585556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,128,0.002436444370283021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,512,0.0038906666967603895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,64,0.0023706667125225067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,768,0.0129120002190272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,512,0.012402666939629449
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,1024,0.01314755529165268
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,256,0.011903111305501727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,32,0.0021768889079491296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,128,0.01165688865714603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,64,0.011586666935020022
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,12288,0.030047112041049536
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,16384,0.0388355553150177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,32,0.011582222249772815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,65536,0.14719200134277344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,10240,0.011149333582984077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,8192,0.009495110975371467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,16384,0.03489333391189575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,7168,0.010523555179437002
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,12288,0.029330667522218492
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,65536,0.11882044209374322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,6144,0.00943999985853831
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,10240,0.026363554928037856
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,8192,0.02311022248533037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,4096,0.007479111353556315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,5120,0.00834222220712238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,7168,0.022097777989175584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,6144,0.020403555697864957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,3584,0.007028444773621029
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,3072,0.00647199981742435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,2560,0.0058808889653947614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,2048,0.005608888963858287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,4096,0.017419555120997958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,3584,0.01688088807794783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,3072,0.0162435554795795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,5120,0.018930666976504855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,2560,0.015372445185979208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,1024,0.005695111221737332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,2048,0.014553777045673795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,768,0.004824888788991504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,1536,0.007660444411966536
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,256,0.0029120000286234748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,512,0.003814222084151374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,1024,0.01313066648112403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,128,0.0025271111064487034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,1536,0.013815999858909182
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,768,0.01273155543539259
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,256,0.011920000116030375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,512,0.012373333175977072
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,64,0.0022337778160969415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,32,0.002159111201763153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,16384,0.014948444234000312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,128,0.011695110963450538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,32,0.011309333145618439
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,64,0.01163200040658315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,12288,0.007889777421951294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,65536,0.10050666994518703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,16384,0.03481511274973551
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,10240,0.007053333024183909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,65536,0.11087466610802545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,8192,0.006548444430033366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,7168,0.006274666637182236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,12288,0.029359999630186293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,6144,0.00701155596309238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,10240,0.026375111606385972
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,7168,0.021853332718213398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,4096,0.005753777921199799
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,5120,0.0064479998416370815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,3072,0.005180444568395615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,5120,0.018996444013383653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,3584,0.005630222045713001
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,4096,0.017391110459963482
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,8192,0.023170666562186346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,3584,0.01684711045689053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,2560,0.005024888863166173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,2048,0.004556444370084339
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,1536,0.007511999871995714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,3072,0.01608800060219235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,1024,0.005631999837027655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,2560,0.015318221516079374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,6144,0.020544888244734872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,2048,0.014538667268223233
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,768,0.004683555414279302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,1536,0.013921777407328287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,512,0.0037235555549462638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,256,0.002798222212327851
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,128,0.0022400000856982335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,1024,0.013013333082199097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,768,0.012801777985360889
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,64,0.002159111201763153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,512,0.012496888637542725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,256,0.011897777517636618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,32,0.002163555473089218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,128,0.01163999984661738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,64,0.011519999967681037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,65536,0.050795555114746094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,16384,0.006890666567616993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,32,0.011323555476135679
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,12288,0.006259555617968242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,10240,0.007027555671003129
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,8192,0.006478222293986215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,16384,0.03490666548411051
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,65536,0.10571555296579997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,7168,0.005741333381997214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,10240,0.026148445076412622
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,12288,0.02938488788074917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,8192,0.02314666741424137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,6144,0.005249777601824866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,5120,0.005103999955786599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,7168,0.02195288903183407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,4096,0.004612444589535396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,3584,0.004919111314747068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,6144,0.02036622166633606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,3072,0.004473777694834603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,5120,0.01903377804491255
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,2560,0.0047048889100551605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,4096,0.017343999611006845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,2048,0.003994666867785984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,3584,0.01685244507259793
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,1536,0.003998222036494149
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,3072,0.01591466698381636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,2560,0.015392889579137167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,1024,0.005481777919663324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,768,0.004536889079544279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,2048,0.014498665928840637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,512,0.0036231109665499795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,256,0.0026862221873468826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,1536,0.013943110903104147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,128,0.002263111165828175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,1024,0.013064000341627332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,768,0.012623111406962076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,512,0.012335110869672565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,64,0.0021626665774318907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,32,0.0021262222694026097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,256,0.01184177812602785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,65536,0.019903999235894945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,128,0.011567999919255575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,16384,0.006955555743641323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,64,0.011613333390818702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,32,0.011216888825098673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,12288,0.005879111174080107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,10240,0.005980444451173146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,8192,0.005445333404673471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,16384,0.03488355543878343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,65536,0.10342133045196533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,7168,0.004893333133724001
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,12288,0.02923111120859782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,6144,0.004771555463473002
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,5120,0.004846222284767362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,8192,0.02314844396379259
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,4096,0.004569777597983678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,7168,0.021701332595613267
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,10240,0.026399110754330952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,6144,0.02032711108525594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,5120,0.01889244384235806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,3072,0.0041582224269707995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,3584,0.004876444323195351
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,2560,0.004469333423508538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,4096,0.017476444443066914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,2048,0.0037208890749348533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,3584,0.016758221719000075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,1536,0.00370755543311437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,3072,0.01605422298113505
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,1024,0.0034604445099830627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,2560,0.015331554743978711
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,768,0.00452533322903845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,2048,0.014519110321998596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,512,0.0035742223262786865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,256,0.0028204443968004654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,1536,0.013991110854678683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,1024,0.012999999854299756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,128,0.002238222294383579
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,768,0.012707555459605323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,64,0.002161777681774563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,512,0.012263110942310758
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,32,0.0021119999388853707
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,256,0.011879111329714457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,65536,0.016190222567982145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,16384,0.006700444552633498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,128,0.011552000211344825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,64,0.011414222419261932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,12288,0.005683555371231503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,32,0.011547555526097616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,10240,0.005787555542257097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,8192,0.0053004444473319584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,16384,0.03475022315979004
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,65536,0.10308000114228989
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,12288,0.02905333373281691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,7168,0.0047066667013698155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,10240,0.026162667406929865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,8192,0.022980444961124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,6144,0.004446222136418025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,5120,0.004315555509593752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,4096,0.004488888714048597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,7168,0.021793777743975323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,6144,0.020023110840055678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,3584,0.004443555656406614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,5120,0.018770666586028207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,3072,0.0040702223777771
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,4096,0.01722044414944119
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,2560,0.0041306668685542215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,2048,0.0038568890757030914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,1536,0.0036106668412685394
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,3584,0.01663733356528812
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,3072,0.01573422220018175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,1024,0.0034266666819651923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,2560,0.015244444211324057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,2048,0.014311111635631986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,768,0.005022222383154763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,512,0.003915555361244413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,1536,0.01371022231049008
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,256,0.002768888862596618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,1024,0.012914666699038612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,128,0.0022151112142536375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,64,0.0021422221842739317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,768,0.01274400038851632
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,512,0.012134222520722283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,256,0.011589333415031433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,32,0.0021182222084866627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,128,0.011182222101423474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,65536,0.016050666570663452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,16384,0.006606222026877933
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,64,0.011439999772442712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,12288,0.00555377784702513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,32,0.011420444481902652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,10240,0.00565155554148886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,16384,0.032877332634396024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,8192,0.005208000126812193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,7168,0.005187555733654234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,65536,0.09617066383361816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,12288,0.027975112199783325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,6144,0.004934222333961063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,10240,0.025107555919223364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,5120,0.004658666749795278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,4096,0.004389333228270213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,3584,0.004283555679851108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,8192,0.02235288918018341
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,7168,0.020837333467271592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,6144,0.01960444450378418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,5120,0.018244443668259513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,3072,0.004088888979620404
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,2560,0.003991999973853429
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,4096,0.016889777448442247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,2048,0.003814222084151374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,1536,0.008399110701349046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,3584,0.016369778248998854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,3072,0.015611555841233997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,1024,0.006161777923504512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,2560,0.014999111493428549
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,768,0.005095110999213325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,2048,0.014117333624098035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,512,0.003926222109132343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,1536,0.01348888874053955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,256,0.0028044444819291434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,128,0.002209777840309673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,1024,0.0125591109196345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,768,0.012462221913867526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,64,0.002104000084929996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,512,0.012096000214417776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,32,0.0020684444655974707
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,256,0.011566222541862063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,128,0.011391999820868174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,64,0.011413333316644033
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,32,0.011375111010339526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,10240,1.1704702377319336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,12288,1.4129262500339086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,8192,1.5332942538791234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,8192,0.9037715064154731
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,16384,1.8516400655110676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,10240,1.866015116373698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,7168,0.7596800062391492
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,12288,2.391204410129123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,5120,0.5132097668117946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,7168,1.3113768895467122
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,6144,0.6833804448445638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,6144,1.1398213704427083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,5120,0.9395928912692599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,16384,2.978893280029297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,4096,0.44375022252400714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,3584,0.36263200971815324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,4096,0.753967973921034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,3584,0.6619902186923557
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,3072,0.31690933969285756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,2560,0.4758186870151096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,3072,0.5684382120768229
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,1536,0.28877777523464626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,2560,0.28612889183892143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,1024,0.1943288909064399
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,2048,0.4137093226114909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,768,0.1463164488474528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,1024,0.12364621957143147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,2048,0.2362577650282118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,768,0.09968711270226373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,128,0.01143200033240848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,512,0.08435999684863621
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,1536,0.18269066015879312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,64,0.005568000177542369
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,32,0.004080888711743885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,256,0.05240177777078417
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,128,0.08168888754314847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,512,0.10550666517681545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,256,0.08149155643251207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,64,0.08251377609041002
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,32,0.08389421966340806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,12288,0.37081066767374676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,16384,0.48266135321723086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,10240,0.5200773345099555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,12288,0.6190497610304091
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,16384,0.833065774705675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,10240,0.301071114010281
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,7168,0.3297235435909695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,8192,0.23098132345411512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,8192,0.42522666189405656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,7168,0.22185244825151232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,6144,0.1772773398293389
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,5120,0.23693866199917266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,6144,0.3009422090318468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,3584,0.16695733865102133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,4096,0.20812977684868705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,3584,0.10491110881169637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,5120,0.1574079990386963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,65536,1.6902213626437719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,4096,0.12779377566443548
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,2560,0.11823199854956733
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,2048,0.09490666786829631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,2560,0.0842355555958218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,2048,0.06879288620418973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,1536,0.07900177770190768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,3072,0.15696177217695448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,768,0.040955556763543025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,3072,0.09816977712843154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,1024,0.053547554545932345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,1024,0.033696889877319336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,512,0.0103493332862854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,256,0.0047395556337303585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,768,0.014121777481502958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,1536,0.052842666705449425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,256,0.011295999917719098
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,64,0.0029893333299292457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,512,0.012460444536474017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,128,0.004033777862787247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,128,0.010975999964608086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,32,0.002598222138153182
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,32,0.010593777729405297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,64,0.010649777948856354
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,65536,2.9502648247612844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,12288,0.5015066464742025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,12288,0.2819688849978977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,16384,0.3367022143469916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,16384,0.6069537798563639
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,10240,0.416503111521403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,10240,0.21314311027526855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,8192,0.31018487612406415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,8192,0.18956532743242052
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,7168,0.26872976620992023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,7168,0.15011910597483316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,4096,0.15436444017622206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,6144,0.2542204327053494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,65536,1.2465662426418729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,5120,0.2093271149529351
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,5120,0.10912089215384589
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,4096,0.09806044234169854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,3584,0.13430489434136286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,3072,0.11542488469017877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,6144,0.1386853324042426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,2560,0.09613600042131211
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,3072,0.0698453320397271
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,1024,0.03906933466593424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,2560,0.06361955404281616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,1536,0.05978666411505806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,1536,0.03720888826582167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,3584,0.08654133478800456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,512,0.007540444533030192
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,768,0.029930667744742498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,2048,0.08393866486019558
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,1024,0.01311466677321328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,256,0.004267555558019214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,2048,0.05189244283570183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,128,0.0032968889507982465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,768,0.012707555459605323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,512,0.011858666936556498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,64,0.003027555636233754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,32,0.0022115556316243275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,128,0.010211555494202508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,256,0.01090311093462838
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,64,0.009860444400045607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,32,0.010492444038391113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,65536,2.2143262227376304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,16384,0.28062134318881565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,16384,0.47111468844943577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,12288,0.23572799894544813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,12288,0.4330497847663031
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,10240,0.18475022580888537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,8192,0.14406311511993408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,10240,0.32774578200446236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,7168,0.208259556028578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,65536,1.0422986348470051
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,7168,0.12468977769215901
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,6144,0.18420533339182535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,6144,0.11652799447377522
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,5120,0.14922489060295954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,4096,0.11881955464680989
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,8192,0.2680799961090088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,3584,0.06907822026146783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,5120,0.09914666414260864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,3584,0.11574578285217285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,3072,0.0892817775408427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,4096,0.08285511202282377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,3072,0.06371377574072944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,1536,0.046057777272330395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,2048,0.06673244635264079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,2048,0.0444160004456838
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,1024,0.03131022387080722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,2560,0.08392977714538574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,512,0.005728000154097875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,768,0.008093333078755273
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,1536,0.019905777441130746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,2560,0.04907822277810839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,1024,0.012504888905419244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,128,0.003113777687152227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,768,0.012085333466529846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,512,0.011329777538776398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,64,0.0028782222006056043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,256,0.0042035554846127825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,32,0.0022657778528001574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,65536,2.080343034532335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,128,0.01032977799574534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,64,0.009961778091059791
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,32,0.010090666512648264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,256,0.010602666272057427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,12288,0.17042133543226454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,12288,0.3145235644446479
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,16384,0.24401778644985625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,10240,0.23608711030748156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,16384,0.42206488715277773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,8192,0.19527555836571586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,10240,0.1419288847181532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,8192,0.11788800027635361
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,7168,0.10329155789481269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,6144,0.16417422559526232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,7168,0.18559732702043322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,65536,0.8074257638719348
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,5120,0.11825156211853027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,3584,0.08342666758431329
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,3584,0.05611911084916857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,5120,0.07650844256083171
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,6144,0.0946719977590773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,3072,0.0716444452603658
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,4096,0.10145688719219631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,2560,0.060348444514804415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,1536,0.03706399930848016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,2560,0.040242665343814425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,2048,0.04863644308514065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,1024,0.008737777670224508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,768,0.00646311127477222
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,2048,0.029670221938027277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,4096,0.06809600194295247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,1024,0.012485333614879184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,256,0.0037519998020595978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,65536,1.4784142176310222
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,3072,0.05096888873312208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,1536,0.013567110730542077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,768,0.012416000167528788
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,512,0.005069333232111401
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,128,0.0029893333299292457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,256,0.010944888823562197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,64,0.002798222212327851
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,128,0.010414222048388587
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,32,0.010657777388890585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,64,0.010211555494202508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,32,0.002354666590690613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,10240,0.2073448896408081
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,16384,0.33106311162312824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,512,0.011633777783976661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,12288,0.2759866714477539
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,12288,0.14190133412679037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,16384,0.21038222312927246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,10240,0.12311822838253444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,8192,0.17208622561560738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,8192,0.10998488797081842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,7168,0.14566311571333143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,7168,0.08793155352274577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,65536,0.6966648631625705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,6144,0.14434044890933567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,4096,0.054888000090916954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,3584,0.07325955231984456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,4096,0.0940435528755188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,5120,0.07084800137413873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,6144,0.07653422488106622
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,5120,0.11685066752963597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,3072,0.04073866539531284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,2560,0.053114665879143604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,3584,0.04721066686842176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,3072,0.0710008872879876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,2048,0.04295466674698723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,2560,0.0341306659910414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,1536,0.03264000018437704
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,2048,0.015648888217078317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,768,0.006278222219811545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,1024,0.007777777810891469
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,1536,0.01309866706530253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,1024,0.012383999923865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,256,0.003693333516518275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,512,0.00498933345079422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,768,0.011728000309732227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,128,0.002943111169669363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,64,0.0027439999911520216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,65536,1.457767062717014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,32,0.0022986665782001284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,512,0.01111644423670239
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,128,0.010080888867378235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,32,0.010652444428867765
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,64,0.01018044435315662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,256,0.010514666636784872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,12288,0.12149243884616429
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,16384,0.2843982113732232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,12288,0.23766043451097277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,16384,0.16452622413635254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,8192,0.1481928825378418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,7168,0.12445333268907334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,10240,0.10568266444736057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,10240,0.1990586651696099
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,65536,0.6057955423990885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,8192,0.08843466970655654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,6144,0.1106133328543769
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,6144,0.06603199905819364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,4096,0.07238488727145724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,5120,0.05762044588724772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,5120,0.08930310938093398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,3584,0.06318310896555583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,7168,0.08169333140055339
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,3072,0.03609244359864129
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,2560,0.04576622115241157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,3584,0.040906667709350586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,3072,0.06120977799097697
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,4096,0.0498924454053243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,2560,0.02142400046189626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,1536,0.02500533395343357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,1024,0.007104888558387756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,768,0.0058355554938316345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,2048,0.04167644513977898
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,1536,0.012683555483818054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,512,0.004602666530344221
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,1024,0.01148977792925305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,2048,0.013233777549531726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,768,0.01107377807299296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,65536,1.2488187154134114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,256,0.003560888684458203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,512,0.010681777364677854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,128,0.003168888803985384
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,32,0.0022444443570242985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,64,0.0027217778066794076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,128,0.009945777555306753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,64,0.0097884444726838
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,32,0.009863110880057016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,256,0.010185778141021729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,12288,0.10391200251049465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,12288,0.1988684468799167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,16384,0.24097243944803873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,16384,0.13399467203352186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,10240,0.15708088874816895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,10240,0.09010577864117092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,8192,0.12393332852257623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,7168,0.10390222072601318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,6144,0.09307289123535156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,7168,0.06526577472686768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,65536,0.5103404257032607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,5120,0.07486933469772339
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,4096,0.06102044714821709
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,8192,0.07795911365085177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,6144,0.05669777923160129
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,5120,0.04736444354057312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,4096,0.039098666773902044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,3072,0.04634133312437269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,3584,0.053118222289615206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,1536,0.010791111323568555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,2560,0.03860355416933695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,3072,0.018868444694413077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,1024,0.007055111229419708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,2048,0.01258044441541036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,2048,0.031397332747777305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,768,0.005581333405441708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,65536,0.9263155195448133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,512,0.004365333252482944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,1024,0.01129066695769628
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,3584,0.03211466802491082
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,256,0.003640888879696528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,768,0.010987555815113915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,128,0.002838222309947014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,2560,0.013541333377361298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,512,0.010611555642551847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,256,0.00998933364947637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,1536,0.011976888610257043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,128,0.00979644474056032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,64,0.0026888888743188647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,32,0.0021848889688650766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,64,0.009491555392742157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,32,0.00996800015370051
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,12288,0.1595324410332574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,16384,0.10930844148000081
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,12288,0.08595644765430027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,10240,0.11863911151885986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,16384,0.21762488947974312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,8192,0.10029155678219265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,7168,0.08420355452431573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,8192,0.05696800020005968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,6144,0.07420177592171563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,10240,0.07160088751051161
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,5120,0.06041422155168322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,7168,0.05690933598412407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,65536,0.4147777822282579
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,4096,0.049366222487555615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,5120,0.039142221212387085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,3584,0.048376888036727905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,3072,0.03770400087038676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,6144,0.04553866717550489
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,65536,0.7430879804823133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,2560,0.03145066565937466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,2048,0.012816000315878125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,3072,0.013340444200568728
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,4096,0.025975111458036635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,1536,0.009575111170609793
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,1024,0.006364444477690591
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,2048,0.011897777517636618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,2560,0.012611555556456247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,3584,0.014820444915029736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,1536,0.011366222467687396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,768,0.0053386667536364655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,1024,0.010601777997281818
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,256,0.0034337778472238113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,768,0.010411555568377176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,128,0.0030871110243929755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,64,0.002712888850106133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,32,0.002132444539003902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,512,0.004386666748258802
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,128,0.009383999639087254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,256,0.009586667021115622
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,64,0.009113777842786577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,512,0.010064000056849586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,32,0.00960355583164427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,12288,0.1393982172012329
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,16384,0.09671377473407322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,16384,0.17154399553934732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,12288,0.07554399967193604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,10240,0.10450400246514215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,8192,0.08774844143125747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,10240,0.06294222010506524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,8192,0.05076622300677829
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,7168,0.07425422138637967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,7168,0.04603999853134155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,5120,0.053399999936421715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,6144,0.06583644284142388
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,6144,0.040631999572118126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,65536,0.38781155480278867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,4096,0.043644444810019604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,5120,0.03460977805985345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,3584,0.03819377885924445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,3072,0.033399999141693115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,65536,0.6513057814704047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,3584,0.013552000125249227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,2560,0.022447110878096685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,2048,0.012303111453851065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,2560,0.012462221913867526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,1536,0.008629333641793992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,2048,0.01163822246922387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,1024,0.006285333385070165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,768,0.00526577772365676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,4096,0.014529777897728814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,3072,0.013046222428480784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,768,0.01039644413524204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,256,0.0034293333689371743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,512,0.00432977784011099
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,1024,0.010458666417333813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,1536,0.011134222149848938
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,512,0.009866666462686326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,64,0.0026791110220882627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,256,0.009443555441167619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,128,0.009200000100665623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,32,0.0026142222599850762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,64,0.009118222528033787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,128,0.0030746666921509635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,32,0.009356444080670675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,10240,0.09054044220182632
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,12288,0.12050399515363906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,16384,0.14779999521043566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,16384,0.08451200193829006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,12288,0.07178133063846164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,8192,0.07636800077226427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,7168,0.06437600321239896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,10240,0.055120001236597695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,65536,0.3194231192270915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,8192,0.045048889186647206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,7168,0.04081955552101135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,5120,0.046486222081714206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,6144,0.05767822265625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,65536,0.5590391159057617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,4096,0.03826133410135905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,3584,0.03352533446417914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,6144,0.03600355651643541
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,3584,0.013394667042626275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,2560,0.014904000692897372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,3072,0.02951822347111172
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,2560,0.012309333516491784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,3072,0.012807110945383707
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,1024,0.006093333164850871
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,2048,0.011559111376603445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,4096,0.013983999689420065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,5120,0.022152889106008742
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,768,0.005457777943876054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,2048,0.010289777484205034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,1536,0.010959111154079437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,1024,0.01033155537313885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,256,0.0033822223130199644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,512,0.004277333203289243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,768,0.010275555153687796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,512,0.009770666559537252
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,128,0.0027786667148272195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,64,0.0026888888743188647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,256,0.009325332939624786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,32,0.002390222210023138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,128,0.009013333254390294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,32,0.00925866680012809
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,64,0.008991111483838823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,1536,0.008776889079146916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,16384,0.12373244762420654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,12288,0.10142933660083348
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,16384,0.07226222091250949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,12288,0.05756533145904541
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,10240,0.08710844649208917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,10240,0.04703377683957418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,8192,0.06535910897784762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,65536,0.2723546557956272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,7168,0.054794665839937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,6144,0.04933511217435201
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,8192,0.0392577780617608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,65536,0.4680595397949219
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,5120,0.04053066505326165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,7168,0.03381244341532389
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,3584,0.028770665327707928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,4096,0.013843555417325763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,4096,0.03250133328967624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,3072,0.01758044461409251
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,6144,0.01732711162832048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,3584,0.013311111264758639
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,5120,0.015229332778188916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,1536,0.008599999878141616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,3072,0.012734221915404001
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,2560,0.012958221965365939
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,2560,0.01216533366176817
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,2048,0.00980533328321245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,2048,0.01145600030819575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,768,0.005741333381997214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,1536,0.010876444478829702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,512,0.0042071110672420925
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,256,0.0033528889632887314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,1024,0.0064071110553211635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,128,0.0027377777215507296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,768,0.010049777726332346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,64,0.0026853332916895547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,1024,0.01033866653839747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,512,0.009678222239017487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,256,0.00923288861910502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,32,0.009298666483826106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,32,0.00231555559568935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,64,0.008752000000741746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,128,0.008932444784376357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,12288,0.08315733406278822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,16384,0.10459378030565049
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,12288,0.04804799954096476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,10240,0.06762311193678114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,65536,0.22308622466193306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,8192,0.054735110865698926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,10240,0.0395688878165351
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,65536,0.3761795626746283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,7168,0.04673511121008131
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,8192,0.02647999922434489
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,7168,0.017602667212486267
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,5120,0.034867554903030396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,6144,0.04100888967514038
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,6144,0.016153777639071148
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,4096,0.022646221849653456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,5120,0.014975999792416891
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,3072,0.0143857780430052
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,16384,0.06582578023274739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,2560,0.012025777664449481
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,4096,0.013791999883121915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,3584,0.013284444808959961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,3584,0.019724445210562814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,3072,0.012616000241703458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,1536,0.008339555727110969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,1024,0.0058319999112023254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,2048,0.011202666494581433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,768,0.004964444372389051
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,2560,0.012028444144460889
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,1024,0.010199999643696679
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,512,0.003980444537268745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,256,0.003256888853179084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,768,0.009904888768990835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,2048,0.009679999616410997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,128,0.002653333254986339
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,1536,0.010832889212502373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,64,0.0026186667382717133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,512,0.009754666851626502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,32,0.0023066666391160754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,256,0.009137777818573846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,128,0.008913777768611908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,64,0.008903111020723978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,16384,0.0772320032119751
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,32,0.009223110973834991
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,12288,0.05785955323113335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,16384,0.047910223404566445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,10240,0.04800800151295132
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,12288,0.038819554779264666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,10240,0.021233777205149334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,65536,0.17494489086998832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,8192,0.04115822248988681
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,65536,0.2841235531700982
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,8192,0.018051554759343464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,5120,0.020773332979944017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,6144,0.03636444608370463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,7168,0.04166133205095927
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,4096,0.016975111431545682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,7168,0.01718933383623759
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,3584,0.015571556157535978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,5120,0.014936000108718872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,3072,0.013396444420019785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,4096,0.013575110998418597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,2560,0.011519999967681037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,3584,0.013008000122176277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,6144,0.016014221641752455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,2048,0.009499555660618676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,3072,0.012531555361217923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,1536,0.007804444266690149
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,1024,0.005896889087226655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,768,0.004850666556093427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,2048,0.011203555597199334
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,512,0.003866666720973121
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,1536,0.010647999743620554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,1024,0.010031110710567897
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,256,0.003056888985964987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,2560,0.01200711147652732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,64,0.00240622212489446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,128,0.0024986666523747975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,768,0.009900444083743626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,32,0.0022231110682090125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,512,0.00958933350112703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,256,0.009172444542249044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,128,0.00888533310757743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,64,0.008792888787057664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,32,0.008903111020723978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,12288,0.03829333186149597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,16384,0.052044444613986544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,10240,0.0328693323665195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,12288,0.022766222556432087
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,65536,0.12058044804467095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,8192,0.010755555497275459
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,16384,0.033061332172817655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,65536,0.19291910860273573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,10240,0.020382222202089097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,7168,0.009184888667530483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,5120,0.007664889097213745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,6144,0.008197333249780867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,8192,0.017998221847746108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,4096,0.006599999964237213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,7168,0.017163554827372234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,6144,0.015976889265908133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,3072,0.013344888885815939
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,5120,0.01482222146458096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,3584,0.013017777767446307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,3584,0.006600889066855113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,2560,0.011332444846630096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,2048,0.009486222432719337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,3072,0.012460444536474017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,4096,0.013563555147912769
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,2560,0.011945777469211154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,1536,0.007644444704055786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,1024,0.005622222191757626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,768,0.0047475554876857335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,2048,0.0111191107167138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,512,0.003768889026509391
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,1536,0.010620444185203977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,256,0.0029084444459941653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,1024,0.010001777774757808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,128,0.0023617777559492323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,512,0.009546666509575313
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,64,0.002285333350300789
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,768,0.009824000298976898
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,256,0.009027555584907532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,32,0.0021537778278191886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,128,0.008912888665994009
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,64,0.008918222453859117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,32,0.009109333157539368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,16384,0.03973599937227037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,12288,0.029496888319651287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,65536,0.1517875591913859
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,16384,0.027095110880004034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,10240,0.010255110760529837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,65536,0.1071386668417189
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,12288,0.02258577777279748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,8192,0.00925244473748737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,7168,0.008807111117574904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,6144,0.00814488861295912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,10240,0.02032711108525594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,8192,0.01798577772246467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,4096,0.006722666737106111
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,5120,0.007496000164084964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,3584,0.006154666758245892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,7168,0.01695644524362352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,5120,0.014742222097184924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,2560,0.005376889059940974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,4096,0.013447110851605734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,6144,0.015841777125994366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,2048,0.00481599983241823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,3072,0.005537777725193236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,3072,0.012410666379663678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,3584,0.013042666845851474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,2560,0.011897777517636618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,1536,0.007463999920421177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,2048,0.0110880004035102
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,1024,0.00553955551650789
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,768,0.004607111215591431
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,512,0.0036657779581016968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,256,0.002850666642189026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,1536,0.01054222219520145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,128,0.0022560000005695554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,768,0.009728000395827824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,1024,0.009933333430025313
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,64,0.002199999988079071
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,512,0.009349333743254343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,256,0.009127111070685917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,32,0.002181333386235767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,128,0.009013333254390294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,16384,0.014613333675596448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,64,0.008784000244405534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,32,0.009296889106432596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,12288,0.009977777798970541
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,65536,0.10161422358618842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,10240,0.00889155517021815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,16384,0.027055111196306016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,65536,0.09609511163499619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,8192,0.007862221863534715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,7168,0.007445333732499017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,12288,0.0225591113169988
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,6144,0.006866666591829724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,10240,0.020280000236299302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,5120,0.006324444380071428
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,8192,0.017907554904619854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,4096,0.005513777749405966
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,7168,0.01700533264213138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,6144,0.015840000576443143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,3072,0.005079111291302575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,5120,0.01475288967291514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,3584,0.005463111317820019
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,4096,0.013441777891582914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,2560,0.005101333475775189
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,2048,0.004456888884305954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,3072,0.01236355553070704
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,1536,0.004455111092991299
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,2560,0.011852444046073489
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,2048,0.011047110789351992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,1024,0.005532444351249271
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,3584,0.012917333179050021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,768,0.004538666870858935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,1536,0.010658666491508484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,512,0.003624888757864634
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,256,0.0027439999911520216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,1024,0.009902222288979424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,128,0.002181333386235767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,64,0.0021235555824306277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,768,0.009709333380063375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,32,0.0020764445265134177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,512,0.009348444640636444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,256,0.009093333449628618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,65536,0.05400355656941732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,64,0.008775110873911116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,16384,0.010381333529949188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,128,0.008973333570692275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,12288,0.006803555621041193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,32,0.008813333180215623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,10240,0.006229333165619109
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,16384,0.02703999976317088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,8192,0.005522666705979242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,7168,0.00674044465025266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,12288,0.022618666291236877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,65536,0.08761866887410481
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,10240,0.02032888929049174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,6144,0.006239111224810283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,8192,0.017927999297777813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,5120,0.005737777799367905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,4096,0.005103999955786599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,7168,0.016968000266287062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,3584,0.00508088908261723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,6144,0.015811555915408664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,5120,0.014764444695578681
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,3072,0.0047386665311124595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,2560,0.004661333229806689
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,4096,0.013404444687896304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,2048,0.003924444317817688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,3584,0.012936000194814471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,1536,0.004102222207519743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,1024,0.003649777836269803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,3072,0.012425777812798819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,2048,0.01105955574247572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,768,0.004521777646409141
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,1536,0.010616888602574667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,512,0.003576888806290097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,2560,0.011877333124478659
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,768,0.009768000079525841
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,1024,0.009859555297427708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,256,0.0027191111197074256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,128,0.002135111019015312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,64,0.002102222293615341
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,512,0.009302222066455418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,256,0.00903200027015474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,32,0.00208177769349681
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,64,0.008599110775523717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,128,0.008931555681758458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,65536,0.01464000013139513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,16384,0.007141333487298753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,32,0.009328000247478485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,12288,0.006224888894293044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,10240,0.005763555566469829
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,8192,0.0052177777720822235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,7168,0.006485333459244833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,65536,0.08101955387327406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,12288,0.02259911100069682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,16384,0.027090667022599116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,6144,0.006048000107208888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,10240,0.020322667227851022
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,5120,0.0046737777690092725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,8192,0.01791111131509145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,4096,0.004276444514592488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,7168,0.01699822147687276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,3584,0.004565333326657613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,6144,0.01577422188387977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,5120,0.014775110615624322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,3072,0.004226666771703296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,4096,0.013456000222100152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,3584,0.012827555338541666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,2560,0.00444088876247406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,2048,0.004013333469629288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,3072,0.01238666640387641
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,1536,0.003968888686762916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,1024,0.0034657776769664553
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,2560,0.01185066666867998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,2048,0.011086222198274402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,768,0.004912000149488449
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,512,0.0037715555065208008
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,256,0.0026986667265494666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,1536,0.010504888991514841
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,1024,0.00998222248421775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,128,0.0020942222326993942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,768,0.009716444545321995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,64,0.0020720000482267803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,512,0.009292444421185387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,32,0.0020657777786254883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,256,0.009120889008045197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,128,0.008836444053384993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,16384,0.0069102222720781965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,64,0.008939555949634975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,65536,0.010106667048401302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,32,0.00923288861910502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,12288,0.005991999887757831
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,10240,0.005531555662552516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,8192,0.005030222237110138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,16384,0.027038221557935078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,65536,0.08100355333752103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,7168,0.005045333256324132
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,12288,0.02258488867017958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,10240,0.02014666630162133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,6144,0.004586666822433472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,5120,0.004549333204825719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,8192,0.017987555927700467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,4096,0.0040755557517210645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,3584,0.004407111141416761
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,7168,0.01700177788734436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,6144,0.015783111254374187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,3072,0.004034666551484002
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,5120,0.014759999182489185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,4096,0.013461333182122974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,2560,0.004367111043797599
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,3584,0.012921777864297232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,2048,0.003572444534964032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,1536,0.008222222328186035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,1024,0.006029333505365584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,3072,0.012410666379663678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,2560,0.011791110866599612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,768,0.004907555464241239
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,2048,0.011139555937714048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,1536,0.010599999792046018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,512,0.003773333297835456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,1024,0.00979911122057173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,256,0.0026862221873468826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,128,0.0020986667109860317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,768,0.009745777481132084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,64,0.002083555484811465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,512,0.009327111144860586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,32,0.0020755556308560902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,256,0.009097778134875828
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,128,0.00868088917599784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,64,0.00871733327706655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,65536,0.009816889133718278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,16384,0.006746666712893381
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,32,0.00926577796538671
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,10240,0.005382222019963794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,12288,0.0058328890138202244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,65536,0.07659200165006849
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,16384,0.025812443759706285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,8192,0.004886222382386525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,12288,0.021599110629823472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,7168,0.004916444420814514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,6144,0.004453333301676644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,10240,0.019519110520680744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,8192,0.01740444368786282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,5120,0.004414222306675381
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,4096,0.00397866674595409
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,7168,0.016353777713245816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,3584,0.01702844434314304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,6144,0.015289778510729471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,3072,0.014893333117167154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,5120,0.014330666926172046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,2560,0.012543111211723752
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,4096,0.01313511116637124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,2048,0.010483555495738983
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,3584,0.012663111090660095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,1536,0.00814488861295912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,3072,0.01202400028705597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,1024,0.006019555446174409
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,2560,0.011560888753996955
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,768,0.004823110997676849
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,2048,0.010947555303573608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,512,0.0037724446091387006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,256,0.0026346666531430352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,1536,0.010363555616802638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,1024,0.009853333234786987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,128,0.0020986667109860317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,768,0.009513777991135916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,64,0.0020639999873108333
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,32,0.0020720000482267803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,512,0.00924444446961085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,256,0.00904177791542477
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,128,0.008808888494968414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,64,0.008904889225959778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,32,0.00886399961180157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,10240,1.163079155815972
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,12288,1.4086169136895075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,8192,1.5231439802381728
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,16384,1.843985875447591
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,10240,1.8551964230007594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,8192,0.8968008889092339
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,7168,0.7533742056952583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,12288,2.3784124586317277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,6144,0.6222133106655544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,7168,1.301030264960395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,5120,0.5094568994310167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,5120,0.931062274509006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,16384,2.962051603529188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,6144,1.2356035444471571
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,4096,0.44386132558186847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,3584,0.3606977727678087
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,3584,0.6550258000691732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,3072,0.31408267550998265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,4096,0.814511087205675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,2560,0.46985334820217556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,3072,0.5620871120029026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,2560,0.28497245576646596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,1536,0.2847226725684272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,2048,0.23452178637186685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,2048,0.4090720017751058
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,1536,0.1815333366394043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,256,0.05229422118928698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,768,0.09786311123106216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,1024,0.20446488592359754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,512,0.08525777525371975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,128,0.011198221809334226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,64,0.005012444323963589
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,32,0.0040755557517210645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,256,0.0794408917427063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,768,0.15429333845774332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,1024,0.1283244424396091
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,64,0.08321688572565715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,128,0.08230221933788724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,512,0.1048906644185384
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,32,0.08343644274605645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,12288,0.336991998884413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,16384,0.47577688429090714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,10240,0.51401334338718
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,12288,0.6777564684549967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,16384,0.8234560224745008
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,10240,0.28734665446811253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,8192,0.24947023391723633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,8192,0.42259377903408474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,7168,0.3277226554022895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,7168,0.2021404504776001
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,6144,0.2884719901614719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,5120,0.14458933141496447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,4096,0.18930310673183867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,4096,0.11798755327860515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,6144,0.19133955902523467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,5120,0.25844889216952854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,65536,1.6295963923136394
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,3072,0.14238311184777155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,2560,0.13067644172244602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,3584,0.18142667081620958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,3072,0.10134755240546332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,2560,0.08489155769348145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,3584,0.11459644635518391
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,1536,0.0787680016623603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,2048,0.10418311092588638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,1024,0.05348089006212023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,2048,0.06775644752714369
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,1536,0.05255022313859728
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,1024,0.03402933478355408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,512,0.012083555261294046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,256,0.004648000001907349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,768,0.040848887628979154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,768,0.013231111069520315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,128,0.00360000009338061
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,64,0.0027173333283927706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,128,0.01091288857989841
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,256,0.011821333732869891
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,32,0.0023893333143658107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,64,0.010562666588359408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,512,0.01239911135700014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,32,0.010218666659461128
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,10240,0.3607066737280951
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,65536,2.947937859429253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,12288,0.2783928977118598
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,16384,0.362319999270969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,16384,0.649411572350396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,12288,0.5093297958374023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,10240,0.210387561056349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,8192,0.30680976973639595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,7168,0.14783555931515163
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,8192,0.17711199654473198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,7168,0.2709733380211724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,6144,0.23828532960679796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,5120,0.1777893304824829
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,4096,0.15369866953955755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,6144,0.1372577746709188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,65536,1.2287555270724826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,4096,0.09080888827641805
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,5120,0.11709600024753147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,3584,0.1445502175225152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,2560,0.09518755806816949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,3072,0.11396443843841553
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,3072,0.06980799966388278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,2048,0.08306222491794162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,1536,0.05778488847944471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,1024,0.03914577762285868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,2560,0.06342399782604642
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,1024,0.012734221915404001
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,1536,0.03719733489884271
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,512,0.005669333454635408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,768,0.03221333358022902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,768,0.012226666841242047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,3584,0.08582133054733276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,128,0.0032435556252797446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,512,0.011424889167149862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,64,0.0029768887907266617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,256,0.010581333604123859
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,128,0.010312889185216691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,256,0.004090666770935059
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,32,0.0022542222092549005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,2048,0.05172622203826904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,65536,2.2125227186414933
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,32,0.009347555538018545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,64,0.01034400032626258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,16384,0.2770328786638048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,16384,0.4659946759541829
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,10240,0.19309600194295248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,12288,0.23267732726203072
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,10240,0.32297688060336643
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,8192,0.24167201254102919
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,12288,0.4269733428955078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,8192,0.1406373315387302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,7168,0.13411022557152644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,6144,0.10724088880750869
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,6144,0.19960355758666992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,5120,0.14809955490960017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,7168,0.22713777754041883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,4096,0.13016977575090197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,4096,0.07531911134719849
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,3584,0.11468711164262559
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,65536,1.026791148715549
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,3072,0.08908889028761123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,2560,0.07484977775149874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,5120,0.09879644711812337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,3072,0.06310222546259563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,2560,0.04977066649330986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,2048,0.06008444229761759
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,2048,0.040361778603659734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,1024,0.03117777903874715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,3584,0.07146844598982069
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,1536,0.05053066545062595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,768,0.007133333219422235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,512,0.00554666668176651
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,1536,0.02141688929663764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,256,0.0038728887836138406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,768,0.01181688904762268
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,512,0.011143999795118967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,128,0.003091555502679613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,64,0.0026542221506436667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,256,0.010633777413103314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,128,0.00996800015370051
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,32,0.0022542222092549005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,1024,0.012458666331238218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,32,0.009586667021115622
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,64,0.010092444717884064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,65536,1.8445440928141277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,16384,0.21931910514831543
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,12288,0.1658302280637953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,12288,0.3122826682196723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,10240,0.23452443546719023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,16384,0.4206079906887478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,8192,0.19552444087134468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,7168,0.16522844632466635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,10240,0.1365857786602444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,7168,0.10077777836057876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,8192,0.12127644485897487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,65536,0.7906159824795194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,5120,0.11793511443667942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,4096,0.0951848891046312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,5120,0.07499377595053779
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,3584,0.0833502213160197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,6144,0.15952089097764757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,3072,0.07167822122573853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,3584,0.054860442876815796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,4096,0.0654711127281189
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,2560,0.060172445244259305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,3072,0.046711110406451754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,6144,0.09312088621987237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,2560,0.039834668238957725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,1536,0.037045333120558
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,2048,0.031870222753948636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,1024,0.014941333068741692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,2048,0.048651556173960366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,768,0.00611200018061532
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,1024,0.0121760004096561
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,256,0.003647110942337248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,512,0.0047884442740016514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,65536,1.4768702189127605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,512,0.011400000088744693
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,768,0.011738667057620155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,1536,0.013456000222100152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,256,0.010747555229398938
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,64,0.0027155555370781156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,128,0.01015022231472863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,64,0.010584000084135266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,128,0.002975111206372579
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,32,0.01016622202263938
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,32,0.0023288888235886893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,12288,0.1405093272527059
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,12288,0.27464267942640513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,16384,0.18902222315470377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,10240,0.11813688278198242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,8192,0.1710853311750624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,8192,0.09849777486589219
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,16384,0.3704711066351996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,10240,0.230841769112481
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,7168,0.16246667173173693
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,65536,0.6850364473130969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,6144,0.12802844577365452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,7168,0.08665155702167088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,4096,0.08368444442749023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,5120,0.06861155562930636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,5120,0.11647289329104954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,3072,0.06326399909125434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,2560,0.05942044655481974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,3584,0.08236089017656115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,3584,0.0466231107711792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,3072,0.04400177796681722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,65536,1.2921466827392578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,4096,0.05634933047824436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,6144,0.08068888717227511
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,2048,0.04292266567548116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,1536,0.03267200125588311
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,2048,0.018374222848150466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,2560,0.034669333034091525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,512,0.004722666823201709
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,1536,0.012755555411179861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,768,0.006028444402747684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,1024,0.01201777739657296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,1024,0.007272000114123027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,256,0.003567999849716822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,128,0.002895111011134254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,64,0.002749333365095986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,512,0.010824888944625854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,32,0.0026586666289303037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,64,0.010121777653694153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,128,0.010141332944234213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,256,0.010630221830474006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,32,0.009735999835862054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,768,0.011376889215575324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,12288,0.23608178562588164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,16384,0.2868995666503906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,12288,0.1318595541848077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,10240,0.1768266624874539
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,16384,0.17753333515591094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,10240,0.10278666681713527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,7168,0.12425066365136041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,8192,0.14784355958302817
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,8192,0.08560533656014337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,7168,0.0758568909433153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,5120,0.08930044704013401
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,6144,0.11020888884862264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,65536,0.6159786648220485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,4096,0.07252977954016791
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,5120,0.05573599868350559
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,3072,0.0547982222504086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,6144,0.07018488645553589
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,3584,0.04030133287111918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,3072,0.035875555541780256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,3584,0.07083021932178073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,2560,0.020037333170572918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,65536,1.1077920065985787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,1536,0.02756799923049079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,4096,0.04895111256175571
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,2048,0.012867555850081973
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,2048,0.04163466559516059
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,1024,0.006861333217885759
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,2560,0.04577155576811897
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,768,0.005722666780153911
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,512,0.004690666579537922
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,1536,0.012056888805495368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,256,0.003479111111826367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,1024,0.011407111254003314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,768,0.011062222222487131
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,64,0.0026862221873468826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,128,0.0028515555378463534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,512,0.010750222537252637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,32,0.002584888910253843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,256,0.01012711144155926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,128,0.009663999908500248
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,64,0.010237333675225576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,32,0.009455111291673448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,16384,0.13743378056420222
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,16384,0.24009778764512804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,10240,0.14801511499616835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,12288,0.10212266445159912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,12288,0.19769244723849824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,8192,0.12443999449412028
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,10240,0.08727555804782444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,7168,0.1040453314781189
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,8192,0.07283377647399902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,7168,0.06548888815773858
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,6144,0.09294488694932725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,6144,0.05614044268925985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,65536,0.5512960222032335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,4096,0.03871377640300327
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,5120,0.07492178016238742
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,3584,0.03513866662979126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,65536,0.9248106214735242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,4096,0.060933331648508705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,3072,0.046366221374935575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,5120,0.04643911123275757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,3584,0.053222222460640796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,2560,0.038690666357676186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,2048,0.031529777579837374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,1024,0.006635555376609166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,3072,0.017772444420390658
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,1536,0.01166311071978675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,2048,0.012359110845459832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,512,0.004354666504595014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,1536,0.013005333642164866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,2560,0.013097777962684631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,1024,0.011054221954610614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,256,0.0034382223255104492
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,128,0.0028364445186323593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,768,0.010789333118332757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,768,0.005569777968857024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,512,0.010473777850468954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,32,0.002537777854336633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,64,0.002570666579736604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,256,0.00998399986161126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,64,0.00981244444847107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,32,0.00907733374171787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,128,0.009755555954244401
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,16384,0.11175733142428929
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,16384,0.1937626732720269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,12288,0.15902933809492323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,10240,0.11862844891018337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,12288,0.08463911215464275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,8192,0.1004871129989624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,7168,0.08432444598939683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,8192,0.05973244375652737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,10240,0.07693599992328219
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,65536,0.4013351069556342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,6144,0.07424710856543647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,7168,0.052694221337636314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,5120,0.060349331961737744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,6144,0.044650667243533664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,4096,0.049617777268091835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,3584,0.04307644565900167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,3584,0.014287999934620328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,5120,0.04267200165324741
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,3072,0.012849777936935425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,4096,0.023552889625231426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,65536,0.7431626849704318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,2560,0.012247111234400006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,2560,0.03151466780238681
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,3072,0.040115555127461754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,1536,0.00833777752187517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,1024,0.006136000156402588
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,768,0.0052133335007561584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,2048,0.014103111293580798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,512,0.004257777912749184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,1536,0.01110311100880305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,1024,0.010468444062603844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,256,0.0033359999457995095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,768,0.010273777776294285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,2048,0.011723555624485016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,512,0.009999111294746399
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,64,0.002483555633160803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,256,0.009656888743241629
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,128,0.009137777818573846
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,32,0.0024871110088295406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,128,0.0027866665687825945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,64,0.00925866680012809
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,12288,0.15666843785179987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,16384,0.09868711233139038
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,12288,0.07579822010464139
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,16384,0.1702035533057319
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,32,0.008896888958083259
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,8192,0.08751555283864339
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,65536,0.3531493345896403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,10240,0.11747999986012776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,7168,0.07422577672534518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,10240,0.06370400057898627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,8192,0.05198577708668179
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,6144,0.065574222140842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,5120,0.05338933401637607
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,65536,0.6495982276068794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,4096,0.04351733459366692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,6144,0.04018488857481215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,5120,0.03446222345034281
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,3584,0.03823555509249369
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,3072,0.033518221643235944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,2560,0.026753778258959453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,3072,0.012450666891203986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,1536,0.008719999757077958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,2560,0.01200799975130293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,3584,0.01313511116637124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,1024,0.006143110907740063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,2048,0.01013422260681788
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,4096,0.014017777310477363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,7168,0.04988444513744778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,2048,0.011401777466138205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,768,0.005142222262091107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,1536,0.010919111470381418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,256,0.003269333392381668
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,512,0.0042151109211974675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,128,0.0027537778433826235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,768,0.010118222071064843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,1024,0.010438222024175854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,64,0.0024275556206703186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,512,0.00981955561372969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,32,0.002437333265940348
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,256,0.009496889180607265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,128,0.009069333473841349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,64,0.009352000223265754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,32,0.008932444784376357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,16384,0.08566933208041722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,12288,0.12047110663519965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,16384,0.1477831072277493
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,12288,0.0667662223180135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,10240,0.09080533186594646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,7168,0.06449955701828003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,10240,0.055148445897632174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,65536,0.30818221304151744
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,8192,0.07667022281222872
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,8192,0.04458311200141907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,6144,0.05782310830222236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,7168,0.040250665611690946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,6144,0.035791112316979304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,4096,0.038451555702421404
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,5120,0.01793244481086731
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,5120,0.05218133330345154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,4096,0.013394667042626275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,3072,0.029579556650585596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,2560,0.013716444373130798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,2048,0.010280888941552904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,65536,0.5840489069620768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,3584,0.037271999650531344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,2560,0.01183733344078064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,2048,0.011408000356621213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,1536,0.010796444283591377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,3584,0.012924444344308643
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,768,0.005128889034191768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,1536,0.008208889100286696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,1024,0.0059279998143513995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,512,0.0041288890772395665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,3072,0.012335999972290464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,1024,0.010201777848932479
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,256,0.003244444520937072
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,768,0.009970666633711921
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,128,0.002749333365095986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,64,0.0026213334252436957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,512,0.009759110709031424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,32,0.0023724445038371617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,128,0.008854221966531541
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,256,0.009297777381208207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,64,0.009259555902745988
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,32,0.008830221990744272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,16384,0.12337511115603977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,12288,0.10362666845321655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,10240,0.07785066631105211
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,16384,0.07033066617117988
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,12288,0.05700977643330892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,10240,0.0462684432665507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,8192,0.06605066855748494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,65536,0.25994666417439777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,6144,0.04908711049291822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,8192,0.03882933325237698
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,7168,0.0343413319852617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,65536,0.4667217996385362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,6144,0.01683200067943997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,5120,0.04161244299676683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,4096,0.0328328874376085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,3584,0.026523555318514507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,5120,0.014449778530332776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,3072,0.014418666561444601
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,7168,0.057681779066721596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,2560,0.012177777787049612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,3584,0.012755555411179861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,2048,0.009487110707494948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,4096,0.013329777452680798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,1024,0.005798222290145026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,2560,0.011832888755533429
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,2048,0.011238222320874533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,768,0.005233777893914117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,1536,0.010663111176755695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,512,0.0040960001448790235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,1536,0.008079999850855933
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,3072,0.012257777982287936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,256,0.0032151111712058387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,768,0.009911999934249455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,128,0.0027475555737813315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,64,0.00231200001306004
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,512,0.009710222482681274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,32,0.0023182222826613323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,128,0.008987555901209513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,256,0.009107555780145857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,64,0.009110222260157267
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,32,0.008462222086058723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,12288,0.08315733406278822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,1024,0.010207999911573198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,16384,0.10458399852116902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,16384,0.05900355842378405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,12288,0.047001779079437256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,10240,0.06715110937754314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,7168,0.046689778566360474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,65536,0.37478844324747723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,8192,0.024136000209384497
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,10240,0.039048890272776283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,6144,0.04127911064359877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,65536,0.2136231131023831
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,7168,0.016537777251667447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,8192,0.05522488885455661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,5120,0.03501510951254103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,4096,0.022478222846984863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,3584,0.015240889456537036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,3072,0.013364444176355997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,2560,0.011536888778209686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,5120,0.014351111319330005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,2048,0.009451555709044138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,3584,0.012702222499582501
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,3072,0.012072000238630505
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,6144,0.01535022258758545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,2560,0.011643555429246692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,1536,0.007755555212497711
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,1024,0.005692444327804778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,4096,0.013073777986897362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,768,0.004896000027656555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,512,0.0038195554580953387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,2048,0.011183111204041375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,1536,0.010626667075686984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,256,0.0033653332955307434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,128,0.0026195556339290407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,1024,0.01014755583471722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,768,0.009838222629494136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,512,0.009370666411187913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,64,0.0023093333260880574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,32,0.0022719999154408774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,256,0.009089777866999308
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,128,0.008913777768611908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,32,0.008402666283978356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,64,0.00923733330435223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,16384,0.07715288798014323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,12288,0.05761688947677612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,10240,0.04770311050944858
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,16384,0.04720888866318596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,12288,0.03773333297835456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,10240,0.02074399921629164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,65536,0.16437333159976536
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,8192,0.04689155684577095
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,65536,0.2828684382968479
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,7168,0.041518221298853554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,6144,0.03581600056754218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,5120,0.026371555195914373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,8192,0.017332444588343304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,7168,0.016248888439602323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,6144,0.015145777000321282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,3072,0.01309866706530253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,4096,0.016413332687483895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,4096,0.01297688898113039
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,2560,0.011194667054547204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,2048,0.009204444785912832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,3072,0.012072888513406118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,3584,0.014975999792416891
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,5120,0.014173333843549093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,3584,0.012640000217490725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,1536,0.00759733302725686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,1024,0.005561777700980504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,2560,0.011617778076065911
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,768,0.0047271110945277745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,2048,0.011010666688283285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,512,0.003823111040724648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,1536,0.010524444282054901
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,256,0.0029582221888833572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,128,0.002428444516327646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,1024,0.009847111172146266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,768,0.009714666340086196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,64,0.002435555474625693
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,512,0.009472889204819998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,32,0.0022213332768943575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,256,0.009094222552246517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,128,0.008822222550710043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,64,0.009291555318567488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,16384,0.05181599987877739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,32,0.008394666843944127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,12288,0.03820533222622342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,16384,0.028619554307725694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,10240,0.03264266583654616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,8192,0.01718933383623759
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,65536,0.1914986636903551
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,10240,0.019280888968043856
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,65536,0.11788266234927708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,12288,0.021357332666714985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,7168,0.0083137775460879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,8192,0.017065778374671936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,6144,0.007629333270920648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,5120,0.006990222467316522
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,7168,0.01622488929165734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,4096,0.00618488879667388
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,6144,0.015160888433456421
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,3584,0.006143110907740063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,3072,0.00554044461912579
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,2560,0.010963555839326648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,4096,0.012960000170601739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,3584,0.012629333469602795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,1536,0.007383999725182851
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,2048,0.009162666896979014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,2560,0.011583999627166323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,1024,0.005480888817045424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,2048,0.010909332997269101
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,5120,0.01422044469250573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,3072,0.011950222154458364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,512,0.0036595554815398324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,256,0.0028320000403457214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,1024,0.009852444132169088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,768,0.004588444613748127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,1536,0.010491555763615502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,768,0.00963466697269016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,128,0.0022764443937275144
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,512,0.009447111023796929
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,64,0.0021626665774318907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,32,0.002132444539003902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,256,0.009071110851234859
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,128,0.008799999952316284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,64,0.009163555171754625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,16384,0.0396382212638855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,32,0.008195555872387355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,12288,0.029535111453798082
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,10240,0.009094222552246517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,16384,0.025136000580257837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,65536,0.14986222320132786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,8192,0.008542222281297049
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,65536,0.10280622376335992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,7168,0.00830488900343577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,10240,0.019183110859658983
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,12288,0.021264889174037512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,6144,0.007469333708286285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,5120,0.007136888802051544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,8192,0.01703999936580658
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,4096,0.006310222049554189
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,3584,0.005604444278611078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,5120,0.014019555515713163
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,3072,0.005116444494989184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,4096,0.012888000243239932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,7168,0.016113777955373127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,3584,0.012524444195959302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,6144,0.015120888749758402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,2560,0.005144888742102517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,2048,0.004601777841647466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,3072,0.011909333368142446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,1536,0.007233777807818518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,1024,0.005403555515739653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,768,0.004517333375083076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,2560,0.01147911118136512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,2048,0.010870222416188983
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,1536,0.010434666441546546
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,512,0.0035946667194366455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,1024,0.009759110709031424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,256,0.002777777819169892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,128,0.0022115556316243275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,768,0.009645333720578088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,64,0.0021280000607172647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,512,0.009312888814343346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,32,0.0021404443929592767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,256,0.009072889056470659
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,128,0.008709333009190029
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,16384,0.015035554766654968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,64,0.00923822240697013
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,32,0.0084915558497111
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,12288,0.009287111461162567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,10240,0.008176888856622908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,8192,0.00740533322095871
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,65536,0.10127200020684136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,12288,0.021215111017227173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,16384,0.025194666451878015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,7168,0.006933333145247565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,65536,0.09213511149088542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,10240,0.0192577772670322
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,6144,0.0064177778032090925
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,4096,0.0052844443255000645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,5120,0.005928888916969299
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,8192,0.017088888419999015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,3584,0.005191111316283544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,6144,0.01512888901763492
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,3072,0.004775111046102312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,4096,0.012914666699038612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,5120,0.01401511165830824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,7168,0.01607822212908003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,2560,0.004775111046102312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,2048,0.004271999829345279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,1536,0.004271111140648524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,3584,0.012501333322789935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,3072,0.012044444680213928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,1024,0.005335111171007156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,2560,0.011492444409264458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,2048,0.010797333386209277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,1536,0.010419555836253695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,768,0.004439110971159405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,512,0.003524444583389494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,256,0.0027217778066794076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,1024,0.009744889206356471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,128,0.0021359999146726397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,64,0.0020791112134853997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,768,0.009600888523790572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,512,0.009319110876984065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,32,0.0020951111283567217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,256,0.009116444322797988
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,128,0.008810666700204214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,64,0.009008000294367472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,16384,0.010055999788973067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,65536,0.05380710959434509
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,12288,0.00833511104186376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,10240,0.0057955553962124725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,32,0.008248888783984715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,8192,0.005231110999981563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,16384,0.02519911030928294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,12288,0.021161778105629817
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,7168,0.006416888700591193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,65536,0.08212355772654216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,6144,0.006015111174848344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,5120,0.005492444253630108
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,4096,0.004917333523432414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,8192,0.017039111918873258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,10240,0.01921688848071628
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,7168,0.01606666710641649
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,6144,0.015044444137149386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,5120,0.014101333088344999
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,3072,0.004289777742491828
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,3584,0.004911111046870549
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,4096,0.012837332983811697
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,2048,0.004012444367011388
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,1536,0.003991111285156674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,3072,0.011935999823941125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,3584,0.012519111235936483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,2048,0.010880889164076911
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,1024,0.003535110917356279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,2560,0.004467555632193883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,768,0.004761777818202972
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,1536,0.010446222292052375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,512,0.00370133337047365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,2560,0.01147555559873581
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,1024,0.009767110976907942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,256,0.0026444445053736367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,128,0.0021084443562560612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,768,0.009569777382744683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,512,0.009290667043791877
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,64,0.002061333300338851
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,32,0.0021102221475707158
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,256,0.008930666579140557
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,128,0.008719999757077958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,65536,0.013640888863139682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,64,0.009199110998047723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,16384,0.006806222101052602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,32,0.008272888759771982
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,12288,0.00591822216908137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,10240,0.005432888865470886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,16384,0.02514044443766276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,65536,0.07393866777420044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,12288,0.02122666603989071
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,7168,0.004949333353175057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,10240,0.019194665882322524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,8192,0.004947555561860402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,6144,0.004540444248252445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,5120,0.004499555461936527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,8192,0.016912889149453905
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,7168,0.01613066593805949
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,4096,0.0047733332547876565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,3584,0.0047688889834615915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,6144,0.015045333239767285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,3072,0.004311111238267687
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,5120,0.014102222190962898
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,4096,0.012879999975363413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,2560,0.004331555631425646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,3584,0.012554667062229581
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,2048,0.0038888889054457345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,3072,0.011888888974984487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,1536,0.00794577764140235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,1024,0.0058542220956749385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,768,0.0047591109242704184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,2560,0.01148799972401725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,2048,0.010810666614108615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,512,0.0037297776175869834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,1536,0.010422222316265106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,256,0.0026311110705137253
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,128,0.0021111110432280437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,1024,0.00980177770058314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,768,0.009591999981138442
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,64,0.0020666666742828158
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,32,0.0020853332761261198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,512,0.009174221919642555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,256,0.00903733323017756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,128,0.008825778133339351
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,64,0.009286222358544668
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,16384,0.006627555522653792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,65536,0.009660444325870937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,12288,0.005745777653323279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,32,0.008359111017651027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,10240,0.005285333428117964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,8192,0.004848888764778773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,16384,0.02509422269132402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,65536,0.073944886525472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,7168,0.004795555439260271
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,12288,0.021207110749350652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,6144,0.004401777767472797
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,8192,0.01701955497264862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,10240,0.0192248887485928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,5120,0.004322666674852371
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,4096,0.003906666818592283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,7168,0.016036444240146212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,3584,0.016399110356966656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,6144,0.015055111712879606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,3072,0.014322666658295525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,5120,0.01407199932469262
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,2560,0.01218311074707243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,4096,0.012850667039553324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,3584,0.01236533373594284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,2048,0.010073777702119615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,1536,0.00815288888083564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,1024,0.005807999935415056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,3072,0.011882666912343768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,2560,0.011431111229790581
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,768,0.004755555755562252
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,2048,0.010873777998818291
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,512,0.0036924444139003754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,1536,0.010363555616802638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,256,0.002622222320901023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,128,0.0020871110674407748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,1024,0.009743111001120673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,64,0.002061333300338851
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,768,0.00943288869327969
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,32,0.00208177769349681
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,256,0.009092444347010719
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,65536,0.00924444446961085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,128,0.0086666668454806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,64,0.008996444443861643
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,16384,0.006599111275540457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,32,0.008454221818182204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,12288,0.005626666463083691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,512,0.009305777649084726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,10240,0.005220444252093633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,8192,0.0047626665068997275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,65536,0.07332355446285672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,16384,0.02493688960870107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,12288,0.02127555509408315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,7168,0.031176000833511353
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,10240,0.019070222973823547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,6144,0.02667466633849674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,8192,0.0169831116994222
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,5120,0.022709333234363135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,4096,0.018287110659811232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,7168,0.01594311164485084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,6144,0.014932443698247274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,3584,0.016388444436921012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,3072,0.014115555418862237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,5120,0.014003554979960123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,2560,0.012154666913880242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,4096,0.01292799992693795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,2048,0.010052444206343757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,3584,0.012258666257063547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,3072,0.011787555283970304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,1536,0.007932444413503012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,1024,0.005744888964626525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,2560,0.011353777514563667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,768,0.004753777964247597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,512,0.003657777690225177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,2048,0.010784000158309937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,1536,0.010336000058386061
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,1024,0.009721777505344814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,256,0.002616000051299731
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,768,0.00941777808798684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,128,0.002096000024014049
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,64,0.002052444343765577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,32,0.0020417778028382198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,512,0.009348444640636444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,256,0.009034666750166152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,128,0.008811555802822113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,64,0.008973333570692275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,32,0.008360000120268928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,10240,1.164979510837131
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,12288,1.406396441989475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,8192,1.519424862331814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,16384,1.8379351298014324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,10240,1.8483369615342882
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,8192,0.8973315556844076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,7168,0.7532551023695203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,12288,2.389887915717231
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,6144,0.6215448909335667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,6144,1.126244439019097
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,7168,1.2956408394707573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,16384,2.955026626586914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,4096,0.4014426602257623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,5120,1.0139591428968642
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,5120,0.5545413229200575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,4096,0.7427502208285861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,3584,0.6509440210130479
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,2560,0.4665439923604329
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,3072,0.3357751104566786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,2560,0.2635822296142578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,3584,0.3898497687445746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,1536,0.28200356165568036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,2048,0.4069937864939372
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,2048,0.23324444558885363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,3072,0.6088826921251085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,512,0.09450578027301365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,1024,0.20661777920193145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,1024,0.12084533108605279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,768,0.15290489461686876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,256,0.05220533410708109
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,128,0.006903111106819577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,512,0.08336533440483941
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,64,0.00488800017370118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,32,0.004062222109900581
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,64,0.08276444673538208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,1536,0.178927116923862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,256,0.08061777883105807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,32,0.07958577738867866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,768,0.09993422031402588
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,128,0.08293955855899386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,12288,0.3318097856309679
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,10240,0.30425333976745605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,16384,0.4748355547587077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,10240,0.5122710863749186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,12288,0.6272302203708225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,8192,0.3837751017676459
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,16384,0.8195137977600098
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,8192,0.22723733054267037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,7168,0.219504886203342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,6144,0.2871849007076687
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,5120,0.14671377340952554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,7168,0.35925955242580837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,6144,0.19061244858635795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,4096,0.12101511160532634
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,5120,0.25733688142564565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,3584,0.1649342245525784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,4096,0.20647911230723062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,3072,0.14169689019521078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,3584,0.11719644069671631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,2560,0.13040799564785427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,2048,0.10411022106806438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,2048,0.06278133392333984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,2560,0.0837893353568183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,65536,1.7500559488932292
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,3072,0.1032186680369907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,1024,0.04835022158092923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,1536,0.07877244551976521
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,768,0.0369137790468004
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,512,0.012512000070677863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,1024,0.029715554581748113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,1536,0.048130667871899076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,256,0.004492444296677907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,128,0.0031999999450312722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,768,0.013226666384273104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,256,0.011244444383515252
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,64,0.0026906666656335196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,32,0.0023653333385785422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,64,0.010204444328943888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,512,0.012528888881206512
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,32,0.009407111340098912
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,128,0.010919999745157031
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,12288,0.46997865041097003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,16384,0.5869768990410699
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,16384,0.3601920074886746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,65536,2.947327084011502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,12288,0.27812621328565806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,10240,0.20913778411017525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,10240,0.35629955927530926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,8192,0.30310310257805717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,65536,1.2200675540500217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,7168,0.286228444841173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,8192,0.18756888972388375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,6144,0.23199910587734648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,7168,0.16163378291659886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,4096,0.15632711516486272
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,3584,0.14429956012301975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,5120,0.2062071164449056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,6144,0.13858755429585776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,5120,0.11828088760375977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,3072,0.11392177475823297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,3072,0.0738453335232205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,3584,0.0865137775739034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,2560,0.10040533542633057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,2560,0.058259553379482694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,4096,0.09887733062108357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,1024,0.03919466667705112
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,2048,0.07638044489754571
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,1536,0.06280710962083605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,2048,0.05121244324578179
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,1536,0.03711822297837999
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,1024,0.012655110822783576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,512,0.005562666803598404
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,256,0.004016889052258598
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,128,0.003195555466744635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,768,0.02479466630352868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,512,0.011317333413494958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,65536,2.2117226918538413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,64,0.0027644443843099806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,32,0.002712888850106133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,256,0.01054488867521286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,32,0.008849778109126622
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,64,0.00962577760219574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,128,0.01072266697883606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,768,0.012332444389661154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,12288,0.23321954409281412
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,16384,0.2765422132280138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,12288,0.38860532972547746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,10240,0.29192090034484863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,16384,0.4651510980394151
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,10240,0.1752986643049452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,7168,0.12288888295491536
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,7168,0.20560710959964326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,6144,0.18194133705563018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,8192,0.14627288447486028
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,8192,0.2633422215779622
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,6144,0.1083822250366211
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,5120,0.09234755569034153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,4096,0.11794488959842259
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,4096,0.07593866851594713
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,3072,0.08899911244710286
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,5120,0.1623200045691596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,65536,1.020345793830024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,3584,0.1143617762459649
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,2560,0.07477777534061007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,2560,0.04872177706824409
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,2048,0.060144000583224826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,3584,0.06691822078492907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,1024,0.031153778235117596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,768,0.007035555938879649
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,512,0.00535377777285046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,3072,0.06254755788379245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,1024,0.012255111502276527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,1536,0.019698666201697457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,1536,0.050534221861097545
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,768,0.011674666570292579
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,2048,0.04417511158519321
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,128,0.002797333316670524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,256,0.0038337777886125776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,64,0.0026435556097163092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,32,0.002640888922744327
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,128,0.010291555689440833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,512,0.011272889044549731
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,64,0.00961866643693712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,256,0.01051288925939136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,32,0.008864888714419471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,65536,1.8435289594862196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,12288,0.16408267286088732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,10240,0.23376711209615073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,16384,0.3740035427941217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,16384,0.23515555593702528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,12288,0.31143376562330455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,10240,0.13593067063225642
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,8192,0.11347022321489121
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,8192,0.19469066460927328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,7168,0.16488088501824275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,7168,0.1018506685892741
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,5120,0.11784088611602783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,5120,0.07498488823572795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,6144,0.1455884509616428
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,4096,0.1069271100891961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,6144,0.09520444605085586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,3584,0.0831751094924079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,65536,0.8267057736714681
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,3584,0.05361244413587782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,3072,0.071743110815684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,2560,0.0395706660217709
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,3072,0.04649066593911913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,2048,0.048588444789250694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,1536,0.03701688845952352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,65536,1.4762133492363825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,2560,0.06008978022469414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,2048,0.030008888906902734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,768,0.006176888942718506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,1024,0.008171555896600088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,4096,0.06446399953630236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,512,0.004744889007674323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,1536,0.013117333253224691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,768,0.011931555966536203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,512,0.011169777976142036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,64,0.002708444371819496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,128,0.002761777697337998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,128,0.010694222317801582
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,256,0.010752889017264048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,256,0.0036115555299652945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,1024,0.012424888710180918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,32,0.002699555622206794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,32,0.009447111023796929
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,64,0.009922666682137383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,16384,0.2058711051940918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,12288,0.13858666684892443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,16384,0.3288266658782959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,12288,0.27400443289015025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,10240,0.23051555951436362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,7168,0.14458844396803114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,8192,0.09781955348120795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,10240,0.11796089013417561
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,8192,0.17069155640072295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,6144,0.12842310799492732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,7168,0.0935128861003452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,5120,0.10352177752388848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,6144,0.07583022117614746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,4096,0.0838773316807217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,5120,0.06408355633417766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,4096,0.0521671109729343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,3584,0.08221422301398383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,2560,0.05295555459128486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,65536,0.7554648717244467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,3072,0.04033066829045614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,2048,0.04820444517665439
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,1536,0.0365866654449039
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,1024,0.007263999846246507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,2048,0.015644444359673392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,3072,0.07082400057050917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,2560,0.03458666801452637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,768,0.006051555689838197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,512,0.004763555609517627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,3584,0.04974400003751119
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,65536,1.2914497587415907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,1024,0.011854222251309289
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,512,0.010964444114102257
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,768,0.011225777367750803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,128,0.002887111157178879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,256,0.010370666782061258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,256,0.0035386664999855887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,1536,0.01257333325015174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,32,0.0026337777574857077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,64,0.009484444227483537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,64,0.0026506665680143568
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,32,0.008925333619117737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,128,0.010321777727868822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,16384,0.1610364384121365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,10240,0.1762862205505371
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,12288,0.13174933857387966
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,16384,0.2849537796444363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,12288,0.23556176821390787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,8192,0.1481599940194024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,10240,0.11056266890631782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,6144,0.1104017761018541
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,8192,0.0848533312479655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,7168,0.07473155524995592
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,6144,0.0655591090520223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,7168,0.13998044861687556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,4096,0.07229422198401557
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,5120,0.055848002433776855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,4096,0.04509777824083964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,5120,0.1002639995680915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,65536,0.5843448638916016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,3584,0.07088622119691637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,3072,0.03567288981543647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,2560,0.051226665576299034
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,2560,0.019652444455358718
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,3072,0.06131733126110501
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,2048,0.01257333325015174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,65536,1.1072195900811088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,3584,0.04398399922582838
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,1024,0.006807111203670502
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,2048,0.04163555635346307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,512,0.004587555511130227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,1536,0.011895111037625207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,1024,0.011181332998805575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,1536,0.023863999380005732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,256,0.0034755555291970572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,768,0.010847110715177325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,128,0.0026106666773557663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,512,0.010620444185203977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,32,0.002575111058023241
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,64,0.00259911103381051
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,768,0.005712888720962737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,256,0.009992000129487779
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,64,0.0092275556590822
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,128,0.01019022199842665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,32,0.008925333619117737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,16384,0.13330933782789442
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,12288,0.1974853277206421
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,12288,0.103057775232527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,10240,0.1478453344768948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,10240,0.08673155307769775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,16384,0.26914845572577584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,8192,0.12347111437055801
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,7168,0.10435733530256484
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,8192,0.07047644588682386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,65536,0.4892017576429579
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,7168,0.06304799848132663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,5120,0.07487110959159003
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,6144,0.1038186682595147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,4096,0.06097955836190117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,6144,0.05526666508780586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,3584,0.053119109736548535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,4096,0.03856888744566176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,5120,0.04699022240108914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,3072,0.04622044497066074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,2048,0.03161688976817661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,65536,0.9239003923204211
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,2560,0.012688889271683164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,1024,0.006502222269773483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,1536,0.011406222151385413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,3584,0.037167112032572426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,3072,0.020767110917303298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,768,0.005355555564165115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,2560,0.03877066572507223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,512,0.004295111116435793
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,1024,0.010949333508809408
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,256,0.0033893332713180115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,2048,0.012103111379676394
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,128,0.0026382222357723448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,1536,0.01148088855875863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,512,0.010249777800507015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,768,0.010758221977286868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,64,0.002528888897763358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,256,0.009719111025333405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,32,0.002520888836847411
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,128,0.010128888818952773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,64,0.009202666580677032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,32,0.008836444053384993
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,12288,0.08437422249052261
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,12288,0.15893777211507162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,16384,0.11822666062249078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,10240,0.11830577585432266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,16384,0.19391022788153756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,8192,0.10002488560146755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,10240,0.07013599740134345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,7168,0.08428355720308091
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,8192,0.06172977553473579
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,65536,0.3952062129974365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,6144,0.07488711012734307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,7168,0.05059733324580722
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,5120,0.06050933069652981
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,6144,0.0444488889641232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,5120,0.03852977686458164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,65536,0.7410737673441569
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,3072,0.03770844472779168
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,3584,0.043097777499092944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,4096,0.024326221810446844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,2560,0.03151022063361274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,2048,0.013069333301650153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,1536,0.008135110967689091
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,4096,0.05610577927695381
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,1024,0.006166222194830577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,1536,0.011021333436171213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,3072,0.012647999657524956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,768,0.005127999931573868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,768,0.010243555737866296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,512,0.004156444635656145
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,3584,0.013396444420019785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,2048,0.011392888923486074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,2560,0.012104888757069906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,256,0.0033200000309281876
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,128,0.0027555556346972785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,1024,0.010478222535716163
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,512,0.009951111343171861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,64,0.0024684444069862366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,32,0.002458666761716207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,256,0.009481777747472128
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,128,0.009278222090668148
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,64,0.008736000292830998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,32,0.008383110993438298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,12288,0.13923911253611246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,16384,0.16961688465542266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,16384,0.09622844722535874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,12288,0.07550755474302503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,7168,0.07423288954628839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,8192,0.050075554185443454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,10240,0.06230311261283027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,8192,0.088319996992747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,7168,0.045016888115141124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,6144,0.06567288769616021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,5120,0.05346666773160299
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,6144,0.039956443839603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,65536,0.34786044226752383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,65536,0.6488151020473903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,10240,0.11707199944390191
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,4096,0.04380089044570923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,3584,0.03820266657405429
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,5120,0.03196622265709771
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,4096,0.013562666873137156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,2048,0.010112889111042023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,3072,0.012416889270146688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,1536,0.00811111099190182
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,2560,0.011774222056070963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,3072,0.03738311264250014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,2560,0.025222222010294598
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,3584,0.012991999586423239
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,1024,0.006035555568006303
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,768,0.005078222188684675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,512,0.0041706665522522396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,1024,0.010216889282067617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,1536,0.010799110763602786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,768,0.010071111222108206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,256,0.0032968889507982465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,512,0.009759110709031424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,128,0.002448888909485605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,64,0.002428444516327646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,2048,0.011300444602966309
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,32,0.002416888872782389
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,256,0.00960355583164427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,128,0.009475555684831407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,64,0.008554666406578487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,32,0.007882666256692674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,16384,0.1464782158533732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,16384,0.08454489045672947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,10240,0.09056178066465591
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,12288,0.1350204414791531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,12288,0.06687111324734159
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,8192,0.07614044348398845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,65536,0.3022711012098524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,10240,0.05420800050099691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,8192,0.044191999567879565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,7168,0.07196711169348823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,7168,0.04000799854596456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,5120,0.04672355453173319
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,6144,0.05761422051323784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,4096,0.04262755645645989
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,6144,0.03339733348952399
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,3584,0.03333244389957852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,65536,0.556988451215956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,5120,0.016068443655967712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,2560,0.012154666913880242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,4096,0.013264888690577613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,3584,0.01276000009642707
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,3072,0.03272799981964959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,2048,0.009944889280531142
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,1536,0.007992000215583378
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,2560,0.011700444751315646
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,768,0.004975111120276981
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,1024,0.0059368887709246735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,512,0.004111999852789773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,3072,0.012283555335468717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,1024,0.010130667024188572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,2048,0.011175110936164856
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,768,0.009886222581068674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,256,0.0032586666444937387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,512,0.009780444204807281
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,128,0.0026746667507621977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,64,0.00240000006225374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,32,0.0023893333143658107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,256,0.009133333133326637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,128,0.009233777721722921
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,1536,0.010712888505723743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,32,0.008092444803979661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,12288,0.10124178065194024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,16384,0.12413956059349908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,16384,0.07221244441138373
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,12288,0.05682577689488729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,64,0.008539555801285638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,10240,0.08685155709584554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,10240,0.04593333270814684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,8192,0.06568621926837497
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,65536,0.25532176759507924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,7168,0.06153421931796604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,6144,0.049125333627065025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,7168,0.037315554089016385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,65536,0.4660657776726617
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,6144,0.016033778587977093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,4096,0.03276000089115567
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,5120,0.01422044469250573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,3584,0.027829332484139338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,3072,0.016833777228991192
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,5120,0.04521155688497755
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,2560,0.011699555648697747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,4096,0.013071999781661563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,2048,0.009463999834325578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,3072,0.012074666718641916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,2560,0.011634666886594562
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,8192,0.04215022259288364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,1536,0.007691555553012424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,1024,0.0057502223385704895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,3584,0.012673777838548025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,1536,0.010592000352011787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,512,0.0038533334930737815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,768,0.0052151112920708126
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,256,0.0032053333189752367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,1024,0.010112000008424124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,128,0.0023831110447645187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,768,0.009866666462686326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,2048,0.011004444625642566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,512,0.009610666996902889
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,64,0.0023182222826613323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,32,0.0022960000981887183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,256,0.00905511114332411
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,128,0.009358222285906473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,64,0.008575110799736446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,32,0.007794666621420119
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,16384,0.10455111000272964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,16384,0.05988622374004788
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,12288,0.08341422345903184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,10240,0.06688355737262301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,12288,0.04684355523851183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,65536,0.20882133642832437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,8192,0.05496711201137967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,10240,0.038719111018710665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,65536,0.3740595446692572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,6144,0.04116800096299913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,5120,0.03486488925086127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,7168,0.046688887808057994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,8192,0.022668444448047217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,6144,0.014948444234000312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,7168,0.016413332687483895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,4096,0.02053955528471205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,3072,0.013036444783210754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,3584,0.0148026661740409
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,2560,0.011136000355084738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,4096,0.012960000170601739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,2048,0.00925866680012809
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,1536,0.007552000383536021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,3584,0.01257955531279246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,2560,0.011459555890825061
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,5120,0.0140657772620519
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,2048,0.011028444601429833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,1024,0.005653333332803514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,768,0.0047288888858424295
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,1536,0.010390222072601318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,3072,0.01202044470442666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,512,0.0037902221083641052
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,128,0.0023066666391160754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,256,0.0031155554784668815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,1024,0.010107555323176915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,64,0.0022657778528001574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,768,0.009885333478450775
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,512,0.009558222360081142
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,32,0.0022640000614855024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,128,0.009149333669079674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,256,0.008880889250172509
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,64,0.008433777425024245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,32,0.008008889026112026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,12288,0.05757066938612196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,16384,0.07694666915469699
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,16384,0.04709511001904806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,12288,0.03761511047681173
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,10240,0.047672000196244985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,65536,0.16029155254364014
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,8192,0.047021332714292735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,7168,0.041300445795059204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,10240,0.02046133412255181
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,8192,0.01681600014368693
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,65536,0.28215734163920086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,6144,0.03637955586115519
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,4096,0.016358221570650738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,5120,0.025549333956506517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,3584,0.014652444256676568
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,7168,0.016003555721706815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,6144,0.014974221587181091
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,5120,0.013976888524161445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,3072,0.012859555582205454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,2048,0.009169778062237633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,3584,0.012442666623327466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,3072,0.011905777785513135
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,2560,0.011414222419261932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,1536,0.007418666448858049
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,2560,0.011060444845093621
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,4096,0.012820444173283048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,1024,0.005524444497293896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,2048,0.010787555740939246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,768,0.0047111109726958806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,512,0.003693333516518275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,1536,0.010450666977299584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,128,0.00240000006225374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,256,0.0029884444342719186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,1024,0.009866666462686326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,768,0.009674666656388177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,512,0.009342222577995723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,64,0.0021662221600612006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,256,0.00906488878859414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,32,0.0022053333620230355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,128,0.009336888790130615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,64,0.008263111114501953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,32,0.00796977761718962
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,16384,0.05184266633457608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,12288,0.03816355599297418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,16384,0.02884355518552992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,8192,0.00925333301226298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,10240,0.03255555695957608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,7168,0.008092444803979661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,12288,0.02081155611409081
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,65536,0.1906977759467231
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,65536,0.11432800028059219
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,6144,0.007393777370452881
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,10240,0.018933332628673978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,8192,0.01679111189312405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,4096,0.005877333382765452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,7168,0.015978667471143935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,3584,0.005985777825117111
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,5120,0.013938667045699226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,3072,0.0052933332820733385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,4096,0.012830221818553077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,5120,0.0068400001360310455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,3584,0.012459555433856117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,2560,0.00536088893810908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,2048,0.009123555488056606
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,6144,0.014819555812411837
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,1536,0.007281777759393056
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,3072,0.011910222470760345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,1024,0.005461333526505365
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,768,0.004629333400064045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,2048,0.010787555740939246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,1536,0.01034400032626258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,2560,0.011491555306646558
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,512,0.0036346668170558084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,1024,0.0097884444726838
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,256,0.0028284444577164124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,128,0.0022640000614855024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,768,0.009631111390060848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,64,0.0021155555215146807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,32,0.0021271111650599376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,512,0.009377777576446533
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,256,0.008990222381220924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,128,0.009365333451165093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,16384,0.0396124455663893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,64,0.008268444074524773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,32,0.007782222496138678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,12288,0.02940355406867133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,10240,0.010025777750545079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,16384,0.024711999628278945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,8192,0.008275555239783393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,65536,0.1435119973288642
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,7168,0.008009778128729926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,65536,0.10108088784747654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,10240,0.01888444523016612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,12288,0.020851555797788832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,6144,0.007189333438873291
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,8192,0.016741333736313712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,5120,0.006956444846259222
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,4096,0.006098666538794835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,3584,0.005485333502292633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,7168,0.01591466698381636
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,3072,0.00499911109606425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,6144,0.014799111419253878
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,2560,0.005076444397370021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,5120,0.01388711151149538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,4096,0.012668444050682915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,3584,0.012383999923865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,2048,0.004534222185611725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,1536,0.00444444434510337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,3072,0.01184800018866857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,1024,0.005392000079154968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,768,0.004498666773239772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,2560,0.011344888971911537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,2048,0.01091288857989841
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,512,0.0037786666717794207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,256,0.0027644443843099806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,1024,0.009707555174827576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,768,0.009553777674833933
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,128,0.0022026666750510535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,1536,0.010235555469989777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,512,0.009266667068004608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,64,0.0021395554973019496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,32,0.0021226666867733
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,256,0.009114666945404476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,128,0.009324444664849175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,16384,0.014541332920392355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,64,0.008529778156015608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,32,0.007829333345095316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,12288,0.00889066689544254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,65536,0.09636888901392619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,10240,0.007859555383523306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,16384,0.024685333172480266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,8192,0.00721688899728987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,7168,0.006776888751321369
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,65536,0.0896951092614068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,12288,0.020853334003024634
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,10240,0.018775999546051025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,6144,0.006275555739800136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,5120,0.0057848890622456866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,8192,0.016760889026853774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,4096,0.005126222140259213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,7168,0.015807999504937064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,3584,0.005109333329730564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,6144,0.014852444330851236
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,5120,0.013810666898886362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,2560,0.004676444249020683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,4096,0.012789333032237159
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,3584,0.012387555506494312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,3072,0.004648000001907349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,2048,0.0042275554604000514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,3072,0.01187999960449007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,1536,0.004181333300140169
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,2560,0.011210666762457954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,1024,0.005361777626805835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,512,0.0036951113078329298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,2048,0.010738666686746808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,768,0.00480355570713679
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,1536,0.010247999595271217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,1024,0.009746666583749983
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,256,0.0026942222482628296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,128,0.0021155555215146807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,768,0.009628444082207149
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,64,0.0020720000482267803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,32,0.0020720000482267803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,512,0.009330666727489894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,256,0.009026666482289633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,128,0.00904799997806549
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,16384,0.010001777774757808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,65536,0.05369955632421705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,64,0.008223111430803934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,32,0.007992000215583378
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,12288,0.008239111138714684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,10240,0.00563466673096021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,8192,0.006799110935793982
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,7168,0.005175999883148405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,16384,0.024783111280865137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,10240,0.018792889184421964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,65536,0.07910222477383085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,6144,0.0058986664646201665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,8192,0.016698666744761996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,5120,0.005427555491526921
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,4096,0.004234666625658671
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,7168,0.015797333584891427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,3584,0.004840888910823398
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,12288,0.02083911167250739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,3072,0.004399111287461387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,5120,0.01386844449573093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,2560,0.004407111141416761
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,4096,0.012723555167516073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,6144,0.014857777290874057
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,2048,0.00396799999806616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,3584,0.012303111453851065
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,3072,0.011733333269755045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,1536,0.007958222594526079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,2560,0.011367110742463006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,1024,0.0058417779703934984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,768,0.0047644442982143825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,2048,0.010804444551467896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,512,0.003719999972316954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,1536,0.010298666854699453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,256,0.0026551110463009942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,1024,0.009764444496896531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,128,0.0020862221717834473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,768,0.009504000345865885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,64,0.0020577777177095413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,512,0.009308444129096137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,32,0.0020844443804687923
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,256,0.0090506664580769
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,65536,0.01071911139620675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,128,0.009188444250159794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,16384,0.006701333241330252
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,64,0.008380444513426887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,32,0.007658667034573025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,12288,0.0058124446206622655
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,10240,0.0053688887920644544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,8192,0.00490399988161193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,7168,0.004873777843183941
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,16384,0.02476444509294298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,12288,0.020847110284699336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,10240,0.01888533267709944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,65536,0.07223911417855157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,6144,0.004459555364317364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,8192,0.0166275550921758
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,5120,0.004439110971159405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,7168,0.01581333412064446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,4096,0.004009777886999978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,3584,0.016399110356966656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,6144,0.014874666929244995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,3072,0.014327110515700446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,2560,0.012186666329701742
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,5120,0.013857777747843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,4096,0.012642666697502136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,3584,0.012255999777052136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,3072,0.01182755579551061
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,2048,0.010058666268984476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,1536,0.007931555310885111
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,2560,0.011424889167149862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,1024,0.005807111246718301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,768,0.0047439999050564235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,2048,0.010819555156760745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,512,0.0036951113078329298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,256,0.0026417778184016547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,1024,0.009730666875839233
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,1536,0.010231110784742568
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,768,0.009591110878520543
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,128,0.0020977778153287037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,512,0.009069333473841349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,64,0.0020746667351987627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,256,0.00888355573018392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,32,0.0020622221959961788
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,65536,0.009327111144860586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,128,0.009222221871217092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,16384,0.006606222026877933
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,64,0.008638222184446123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,32,0.007633777956167857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,12288,0.005694222119119432
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,10240,0.005254222287072076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,8192,0.004777777940034866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,16384,0.024711999628278945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,65536,0.07222400108973186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,12288,0.02084355552991231
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,10240,0.018740443719757926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,7168,0.0311173333062066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,6144,0.026983999543719824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,8192,0.016739555531077914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,5120,0.022690667046440974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,7168,0.01581333412064446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,4096,0.01846933364868164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,6144,0.014789332946141561
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,3072,0.014296000202496847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,3584,0.016388444436921012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,5120,0.01389422184891171
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,2560,0.01220266686545478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,4096,0.01278133359220293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,2048,0.010062221851613786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,3584,0.012134222520722283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,1536,0.00793333351612091
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,3072,0.01181688904762268
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,1024,0.0058373332851462895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,768,0.0047360000511010485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,2560,0.011184888581434885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,512,0.0036951113078329298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,2048,0.010780444575680627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,1536,0.010327999790509542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,1024,0.009578666753239101
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,256,0.002620444529586368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,128,0.0020702222569121257
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,768,0.009451555709044138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,64,0.0020497778637541667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,32,0.002054222135080232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,512,0.00927111092540953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,65536,0.009095999929640029
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,256,0.008839111361238692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,128,0.009341333475377824
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,16384,0.006527110934257507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,64,0.00830488900343577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,32,0.007752888732486301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,12288,0.005579555614127054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,10240,0.00517866677708096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,8192,0.035087111923429705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,65536,0.07102933194902208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,16384,0.02432800001568264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,12288,0.020591111646758187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,7168,0.03109688891304864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,10240,0.018622222873899672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,8192,0.01676977839734819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,6144,0.026945778065257605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,5120,0.02269866731431749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,4096,0.01846844454606374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,7168,0.015657777587572735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,3584,0.016356445021099515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,6144,0.014534221755133735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,3072,0.01424888935354021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,5120,0.01367288910680347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,4096,0.012641777594884237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,2560,0.012014221813943652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,2048,0.010047111246320937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,3584,0.012230222423871359
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,1536,0.007834666305118138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,1024,0.005809777726729711
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,3072,0.011745777395036487
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,2048,0.010793777803579966
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,2560,0.011142222417725457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,768,0.0047457776963710785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,1536,0.010228444304731157
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,512,0.0037173334923055437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,256,0.0026355555488003623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,1024,0.009725333087974125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,128,0.0020942222326993942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,768,0.009531555904282464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,64,0.002040000011523565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,512,0.009099555512269339
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,32,0.0020471111767821843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,256,0.008815111385451423
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,128,0.009298666483826106
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,64,0.008447999755541483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,32,0.007624000310897827
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,10240,1.1633484098646376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,12288,1.400933371649848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,16384,1.8415724436442058
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,10240,1.8451012505425348
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,8192,1.5151600307888453
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,8192,0.8962426715426974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,12288,2.389331605699327
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,7168,0.7528266906738281
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,7168,1.2927271525065105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,6144,0.6213368839687771
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,16384,2.9506746927897134
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,5120,0.5537564489576552
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,6144,1.123321745130751
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,4096,0.7410088645087348
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,5120,0.9246355692545573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,4096,0.4399680031670465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,3072,0.33127199278937447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,3584,0.38903199301825625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,2560,0.4646471341451009
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,2048,0.37272267871432835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,3584,0.7087804476420084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,2048,0.23331732220119902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,3072,0.6071297857496474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,2560,0.28460711903042263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,1536,0.1684044467078315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,1536,0.2803582350413005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,1024,0.1882693370183309
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,512,0.10209688875410293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,256,0.04676889048682319
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,768,0.15572888321346706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,768,0.10085244311226739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,512,0.0833946665128072
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,64,0.006425777657164468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,32,0.004032888760169347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,128,0.0813537769847446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,64,0.07916711436377631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,32,0.07919200261433919
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,1024,0.12759466965993246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,256,0.08088711235258314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,128,0.011933333343929715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,16384,0.47389597362942165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,12288,0.365231990814209
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,10240,0.30356089274088544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,10240,0.5101333194308811
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,12288,0.6708044475979276
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,16384,0.8567191229926215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,7168,0.2010693285200331
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,8192,0.38465332984924316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,8192,0.24806043836805555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,7168,0.35968889130486387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,6144,0.19072088930341932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,6144,0.31479112307230633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,5120,0.14757600095536974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,4096,0.20590133137173125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,3584,0.18072356118096244
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,5120,0.2573475572797987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,65536,1.617794672648112
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,3072,0.15572355853186712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,4096,0.13260889053344727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,2560,0.13034044371710882
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,2560,0.0804684427049425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,3072,0.10431910885704888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,2048,0.09472977452807957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,3584,0.11814044581519233
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,1024,0.04836800032191806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,2048,0.062443554401397705
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,1536,0.07871911260816786
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,1536,0.052095111873414784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,512,0.009100444614887238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,768,0.01348355578051673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,256,0.004451555510361989
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,768,0.04078933265474107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,512,0.012177777787049612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,64,0.0029377777957253983
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,32,0.0028702221396896574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,256,0.011300444602966309
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,1024,0.033548444509506226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,128,0.01035199976629681
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,128,0.0031804444475306403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,64,0.009544889132181803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,32,0.009639110830095079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,16384,0.35843732621934676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,12288,0.4661706818474664
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,65536,2.947120030721029
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,16384,0.6414506700303819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,12288,0.2769377761416965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,10240,0.20843556192186144
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,10240,0.3711511029137506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,8192,0.17151910728878447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,8192,0.2980426682366265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,65536,1.2165315416124132
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,7168,0.2848364512125651
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,7168,0.1623120043012831
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,6144,0.2290088865492079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,5120,0.18570844332377115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,5120,0.11073244280285305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,4096,0.09281244542863633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,3584,0.13014222515953913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,6144,0.14028178320990667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,4096,0.1559280024634467
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,3072,0.1138391097386678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,3072,0.06866666343477038
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,1536,0.05797866980234782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,2560,0.06237866481145223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,2048,0.05107288890414768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,1024,0.039220445685916476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,1536,0.036863999234305486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,2560,0.10402399963802761
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,3584,0.08740711212158203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,768,0.030064000023735896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,2048,0.08289777570300631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,512,0.005502222312821283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,1024,0.012495111260149213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,768,0.012048888537618848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,128,0.00291377781993813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,64,0.0027226667023367355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,512,0.011456888582971362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,128,0.009571555587980483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,65536,2.211639192369249
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,32,0.0026737778551048706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,256,0.003735110991530948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,256,0.011029333704047732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,32,0.009025777379671732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,64,0.008873778084913889
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,16384,0.2744062211778429
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,16384,0.4641146659851074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,10240,0.29131465488009983
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,12288,0.4230088922712538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,12288,0.2332248952653673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,8192,0.14055111673143175
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,8192,0.24012711313035753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,10240,0.18406577905019125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,7168,0.13451377550760904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,6144,0.19809777206844756
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,7168,0.22186044851938883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,5120,0.09346133470535278
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,65536,1.015045377943251
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,6144,0.11784977383083767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,5120,0.16175911161634657
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,4096,0.12964622179667154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,3584,0.10348800155851577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,3072,0.0888942215177748
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,4096,0.07631733020146687
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,3584,0.0657679968410068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,3072,0.05753066804673937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,2048,0.06009778049257067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,2560,0.08324355549282499
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,1024,0.03119644522666931
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,768,0.009099555512269339
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,1024,0.01221422188811832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,512,0.005330666485759947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,1536,0.05052622159322103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,1536,0.018147556318177115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,768,0.011722666521867117
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,2560,0.05279022124078539
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,128,0.0027520000520679685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,256,0.003547555456558863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,64,0.0026506665680143568
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,256,0.010788444843557147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,32,0.0026231110095977783
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,128,0.009484444227483537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,64,0.008968888885445064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,65536,1.8433625962999132
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,512,0.011176889141400656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,32,0.008786666724416945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,2048,0.0441599984963735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,16384,0.21614844269222686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,12288,0.16557333204481336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,16384,0.3736115561591254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,12288,0.31181245379977757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,10240,0.262776878145006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,10240,0.14968978034125432
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,7168,0.16454133722517225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,8192,0.1929182211558024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,7168,0.10360355509652032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,8192,0.11601332823435466
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,4096,0.0952906674808926
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,65536,0.7797813415527344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,6144,0.16327467229631212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,5120,0.07462755839029948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,5120,0.1325413386027018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,4096,0.06028978029886881
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,6144,0.09682044718000625
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,3072,0.07168000274234347
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,2560,0.060071998172336154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,3072,0.045956446064843066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,2048,0.04860622353023953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,2560,0.04296444522009956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,1536,0.0370417767100864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,3584,0.09363644652896458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,768,0.006011555592219035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,1024,0.016385777129067313
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,1024,0.011997333003415002
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,3584,0.05685333410898844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,2048,0.029683556821611192
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,65536,1.4758417341444228
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,512,0.004823110997676849
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,256,0.003705777641799715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,768,0.011752888560295105
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,512,0.011085333095656501
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,128,0.0027253333893087176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,256,0.010852444503042432
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,32,0.0026933333526055017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,64,0.0026968889352348116
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,1536,0.01314933349688848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,64,0.009293333523803288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,128,0.010129777921570672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,32,0.009376889301670922
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,16384,0.18697422080569795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,12288,0.29308533668518066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,16384,0.36402222845289445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,10240,0.11927733156416152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,12288,0.15412978331247965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,10240,0.23042755656772188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,65536,0.6753742429945203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,8192,0.1707315577401055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,7168,0.1444595522350735
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,8192,0.10606666405995686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,6144,0.07587644126680163
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,6144,0.143359104792277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,7168,0.09514666928185357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,5120,0.10347377591662937
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,5120,0.06330400043063693
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,4096,0.08366222514046563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,4096,0.051496889856126576
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,3584,0.0497555567158593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,2560,0.05294933252864414
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,3072,0.07093333535724215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,65536,1.2913102043999565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,3584,0.0824071102672153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,2560,0.036778665251202054
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,1536,0.03277511066860623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,2048,0.0481333335240682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,768,0.005919111271699269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,3072,0.04372266597217984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,512,0.0046453335218959385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,1024,0.0070879997478591064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,2048,0.015472888946533203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,1024,0.011545778148704104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,256,0.0034577778230110803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,128,0.0026791110220882627
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,512,0.01092977739042706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,1536,0.012502222425407834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,256,0.010672888822025724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,768,0.011642666326628791
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,64,0.0026266665922270883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,32,0.0026506665680143568
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,128,0.009735110733244155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,64,0.009016888837019602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,32,0.009185777770148383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,16384,0.1615262164009942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,12288,0.1211164395014445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,16384,0.2848577764299181
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,12288,0.23534133699205187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,10240,0.19805067115359834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,8192,0.14648711681365967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,7168,0.12408799595303005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,8192,0.08550488948822021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,10240,0.10364622539944118
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,4096,0.07268266545401679
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,6144,0.06459999746746488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,7168,0.08080266581641303
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,6144,0.12417066097259521
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,5120,0.09287555350197686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,5120,0.059133330980936684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,3584,0.06310400035646227
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,65536,0.6254622141520182
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,2560,0.04584799872504341
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,2048,0.03717689050568475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,3072,0.06141333447562324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,3072,0.03553955422507392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,3584,0.040192888842688665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,1024,0.006711999989218182
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,2048,0.012487110992272695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,1536,0.028360890017615423
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,4096,0.04465422365400526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,768,0.005666666560702854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,1024,0.011125333607196808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,1536,0.01184177812602785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,65536,1.106843524509006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,512,0.010495999621020423
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,256,0.0035137778355015647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,128,0.0026142222599850762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,256,0.010411555568377176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,2560,0.023152000374264185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,512,0.00452977791428566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,64,0.002597333242495855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,128,0.009401777552233802
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,32,0.002570666579736604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,32,0.00867555538813273
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,64,0.008913777768611908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,768,0.011128000087208219
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,16384,0.13353778256310359
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,12288,0.10262578063540989
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,10240,0.16260533862643772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,12288,0.1937626732720269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,10240,0.08725511365466648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,8192,0.07086488935682508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,8192,0.12369599607255723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,7168,0.11714043882158066
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,65536,0.4847342173258464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,6144,0.09221066368950738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,5120,0.0841368900405036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,4096,0.060884442594316274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,16384,0.2569448947906494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,7168,0.0631964471605089
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,3584,0.05317777726385328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,6144,0.055121776130464345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,4096,0.04225866662131416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,5120,0.050628443559010826
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,3072,0.04628888765970866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,2560,0.038727111286587186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,3584,0.03815022110939026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,65536,0.9233404795328776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,3072,0.018231999542978074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,2048,0.035210665729310774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,1536,0.009908444351620143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,1024,0.006502222269773483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,2560,0.012609778179062737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,2048,0.012050666742854647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,512,0.004326222257481681
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,768,0.00556355549229516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,1024,0.010857777463065254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,1536,0.01148622234662374
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,128,0.002559111143151919
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,512,0.010008888940016428
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,256,0.0032622222271230486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,768,0.010999999940395355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,128,0.00923466682434082
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,32,0.0025351111673646504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,64,0.008567111359702216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,64,0.002528000002106031
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,32,0.008742222355471717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,256,0.010414222048388587
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,16384,0.20657511552174887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,16384,0.10779288742277358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,12288,0.08388000064425999
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,12288,0.15394400225745306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,10240,0.13332088788350424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,8192,0.09907466835445827
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,7168,0.08414577775531346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,10240,0.07027733325958252
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,65536,0.39223821957906085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,8192,0.05814399984147814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,5120,0.06056177616119385
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,7168,0.055705779128604464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,4096,0.049343999889161855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,6144,0.0749173363049825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,5120,0.03844177722930908
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,6144,0.044450667169359
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,3072,0.0377422206931644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,4096,0.025083555115593806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,65536,0.7405955526563855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,3584,0.013912889692518445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,2048,0.01258488910065757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,3584,0.048134220971001514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,2560,0.011893333660231696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,3072,0.012484444512261285
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,2048,0.011499555574523078
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,1024,0.006115555349323485
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,2560,0.0314977765083313
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,1536,0.010860444770918952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,768,0.005040888984998067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,512,0.0041644444896115195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,1536,0.008080888953473832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,256,0.003034666594531801
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,1024,0.010323555933104621
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,64,0.002471111093958219
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,128,0.002489777695801523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,512,0.009944000177913243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,32,0.002477333363559511
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,256,0.00962222201956643
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,128,0.008956444760163626
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,64,0.008368000388145447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,32,0.008105778031879002
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,768,0.0105253333846728
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,16384,0.17272178332010904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,10240,0.10484888818528916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,16384,0.09637333287133111
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,12288,0.13931822776794434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,10240,0.0672631131278144
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,8192,0.08795022302203709
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,7168,0.07421955797407362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,12288,0.07983911037445068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,8192,0.05051111181577047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,65536,0.3436257839202881
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,6144,0.03985066546334161
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,5120,0.03453422254986233
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,5120,0.05982310904396904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,6144,0.06608711348639594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,3584,0.03819200065400865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,7168,0.049606220589743726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,65536,0.6484195391337076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,4096,0.013722666435771518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,4096,0.04934755629963345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,3072,0.0335822237862481
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,2048,0.010030222435792288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,1536,0.008002666963471307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,3072,0.01218311074707243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,3584,0.012830221818553077
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,1024,0.00598933340774642
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,2048,0.011319999893506369
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,768,0.004978666702906291
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,1536,0.010749333434634738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,512,0.0041511112617121804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,256,0.0032426667296224167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,1024,0.010152888794740042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,128,0.002444444431198968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,768,0.01016177816523446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,2560,0.011794666449228922
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,2560,0.024715556038750544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,64,0.002423111142383681
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,128,0.008476444416575963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,256,0.009582222335868413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,32,0.0024337776833110386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,512,0.009698666632175446
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,64,0.00795911086930169
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,32,0.008147555920812819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,16384,0.14239911238352457
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,12288,0.12034666538238525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,10240,0.09057244327333237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,12288,0.06354666418499418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,16384,0.09107733435100979
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,8192,0.07626399728986952
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,65536,0.2973235448201497
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,7168,0.06464444266425239
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,10240,0.05354666709899902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,5120,0.046408888366487294
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,8192,0.044124444325764976
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,6144,0.03534844517707825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,65536,0.5565822389390733
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,7168,0.03995199998219808
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,3584,0.03331911232736375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,4096,0.038633777035607234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,6144,0.057816889550950795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,4096,0.013209777573744455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,3072,0.029333333174387615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,2560,0.013640888863139682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,5120,0.016355555918481614
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,3072,0.012075555821259817
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,2048,0.009935110807418823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,1024,0.005816000203291575
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,2560,0.011706666813956367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,2048,0.011199110911952125
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,1536,0.010729778144094678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,3584,0.012664889295895895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,768,0.004882666799757215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,512,0.004074666649103165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,256,0.0032266666077905228
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,1024,0.010194666683673859
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,768,0.01016355554262797
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,512,0.009713778065310584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,256,0.009200000100665623
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,1536,0.007867555651399825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,128,0.002415111081467734
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,64,0.0023768889821237987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,32,0.0023511110080613033
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,128,0.008475555313958062
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,64,0.008034666379292807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,32,0.007771555748250749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,16384,0.12692533599005804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,12288,0.09566399786207412
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,10240,0.07874666982226901
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,16384,0.06980533070034452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,12288,0.05481333202785916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,8192,0.0654311113887363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,65536,0.2511146598392063
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,7168,0.05479022198253208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,10240,0.04574133290184868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,8192,0.03824977742301093
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,6144,0.04930222365591261
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,65536,0.46599377526177305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,4096,0.03272622161441379
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,6144,0.016527111331621807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,7168,0.03198044498761495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,5120,0.040692445304658674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,4096,0.013017777767446307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,3584,0.027042667071024578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,2560,0.0112631113992797
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,2048,0.009382222261693742
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,3584,0.012410666379663678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,5120,0.014154665999942355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,3072,0.012068444656001197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,2560,0.011533333195580376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,1024,0.005746666755941179
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,1536,0.0075937774446275495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,3072,0.018033777674039204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,768,0.005105777747101254
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,512,0.0038524443904558816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,2048,0.011040888726711273
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,1536,0.010470222267839642
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,256,0.0029306666304667792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,1024,0.010057777994208867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,128,0.0023742222951518162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,64,0.0023066666391160754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,768,0.010106667048401302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,512,0.009532444179058075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,32,0.0022986665782001284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,256,0.009535111486911774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,128,0.00830577810605367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,64,0.008017777568764156
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,32,0.007779555188284979
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,12288,0.08358399735556708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,10240,0.06706666946411133
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,16384,0.10405511326260036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,12288,0.04637866550021701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,16384,0.05824355284372965
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,8192,0.054620444774627686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,65536,0.20585689279768202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,10240,0.03837688763936361
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,6144,0.04125510983996921
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,8192,0.021698666943444148
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,7168,0.04643644558058845
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,5120,0.03482310970624288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,65536,0.3734249009026422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,7168,0.016089777151743572
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,4096,0.022479999396536086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,3584,0.014681778020328946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,5120,0.013914666242069669
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,2560,0.011048888994587792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,3584,0.012443555725945367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,6144,0.014911111858155994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,2048,0.009223110973834991
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,3072,0.01293155550956726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,4096,0.012970666918489667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,2560,0.011446221835083432
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,1536,0.0074551113777690465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,1024,0.005646222167544895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,768,0.0047173334492577445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,512,0.0037662221325768363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,3072,0.011893333660231696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,1536,0.010346666806273991
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,1024,0.010020444790522257
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,768,0.010034666293197209
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,256,0.002879111096262932
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,128,0.0023022221608294383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,2048,0.010946667028797997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,512,0.009327111144860586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,32,0.0022293333378103045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,256,0.009348444640636444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,128,0.008549333446555668
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,64,0.008068444828192392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,32,0.007886221839321984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,64,0.0022693334354294673
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,16384,0.0769760012626648
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,65536,0.15800533029768202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,12288,0.057668447494506836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,16384,0.04651022288534376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,10240,0.047649777597851224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,65536,0.28181065453423393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,12288,0.03743022349145677
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,8192,0.03860177927547031
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,10240,0.019559999306996662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,6144,0.03503733211093479
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,5120,0.020083554916911654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,7168,0.04187377625041538
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,8192,0.016685333516862657
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,7168,0.015863999724388123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,6144,0.014900444282425774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,3584,0.014627556006113688
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,5120,0.013815110756291283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,4096,0.012682666381200155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,4096,0.01633155511485206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,2048,0.009155555731720394
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,3072,0.012749333348539142
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,3072,0.011864888999197217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,1536,0.007334222396214803
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,2048,0.010782221953074137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,3584,0.012355555262830524
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,1024,0.005532444351249271
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,2560,0.011019555230935415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,2560,0.011340444286664328
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,768,0.004615111069546806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,1536,0.01035822182893753
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,256,0.0027946666296985415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,512,0.0038542221817705366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,128,0.0022435554613669715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,1024,0.009672889278994666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,768,0.009910222556855943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,64,0.00221422231859631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,32,0.0021768889079491296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,512,0.009339555270142024
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,256,0.009294222626421187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,128,0.00850311087237464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,16384,0.049545778168572314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,64,0.007992000215583378
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,32,0.007983999947706858
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,12288,0.03809422254562378
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,65536,0.19037777847713896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,10240,0.032510221004486084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,65536,0.11638043986426459
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,8192,0.009711999860074785
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,12288,0.02077511118517982
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,16384,0.02824444572130839
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,10240,0.01868266695075565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,6144,0.007247111035717859
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,5120,0.006717333363162146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,8192,0.016606221596399944
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,7168,0.015654222832785714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,7168,0.007869333028793335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,4096,0.005859555469618903
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,6144,0.01475288967291514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,3584,0.005983110931184556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,3072,0.005264889034960005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,2560,0.005320888840489917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,5120,0.013854222165213691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,4096,0.01273155543539259
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,2048,0.009069333473841349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,3584,0.012245333029164208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,1024,0.005464000006516774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,1536,0.007278222176763747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,2048,0.010808000134097205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,768,0.004868444469239977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,1536,0.010315555665228102
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,3072,0.011844444606039258
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,2560,0.011364444262451597
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,1024,0.009688888986905416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,512,0.003777777983082665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,256,0.0027253333893087176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,128,0.002190222135848469
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,768,0.009750222166379293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,64,0.002130666747689247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,32,0.0021475555582178966
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,256,0.009392889009581672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,512,0.009301332963837517
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,128,0.008220444122950235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,64,0.007807110746701558
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,32,0.007824888659848107
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,12288,0.03135733471976386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,10240,0.009333333207501305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,16384,0.024450666374630396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,12288,0.020617778102556866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,65536,0.14300889439053005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,8192,0.00816000004609426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,65536,0.10101066695319282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,10240,0.018581334087583754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,7168,0.007956444389290279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,6144,0.007164444360468123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,5120,0.006893333461549547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,8192,0.01662400033738878
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,4096,0.006069333189063602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,16384,0.040580444865756564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,3584,0.0053671110007498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,3072,0.005000000198682149
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,6144,0.014727110664049784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,5120,0.013707555830478668
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,4096,0.01262399968173769
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,3584,0.012223111258612739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,2560,0.00498844434817632
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,2048,0.004476444588767158
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,7168,0.015628443823920358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,1536,0.004454222404294544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,3072,0.011805333197116852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,1024,0.0058959999846087555
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,2560,0.011345778074529437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,768,0.004832889056868023
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,2048,0.010681777364677854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,512,0.0037546666959921518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,256,0.002704888789190186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,1024,0.009719111025333405
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,128,0.0021537778278191886
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,1536,0.010312000082598792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,768,0.009875555833180746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,64,0.0021235555824306277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,32,0.002135111019015312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,512,0.00921688891119427
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,256,0.009151111046473185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,128,0.00833422193924586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,64,0.00795022232664956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,16384,0.01456622282663981
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,32,0.008012444608741337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,12288,0.008809777597586313
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,16384,0.02433066732353634
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,65536,0.09631911251280044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,8192,0.007138667007287343
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,10240,0.007792000141408708
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,65536,0.08851199679904515
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,7168,0.006673777682913675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,12288,0.02055199940999349
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,6144,0.006239111224810283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,10240,0.01870577699608273
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,5120,0.005349333501524395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,8192,0.01647911138004727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,7168,0.015623999966515435
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,4096,0.005119110975000593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,3584,0.00509066672788726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,3072,0.004670222186379963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,5120,0.01367200000418557
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,2560,0.00463733325401942
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,4096,0.012647999657524956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,6144,0.01475288967291514
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,3584,0.01221333361334271
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,2048,0.00425155543618732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,1536,0.007977777885066139
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,3072,0.01166133334239324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,2560,0.011223110887739392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,1024,0.005852444304360284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,768,0.0047600000268883174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,2048,0.010696888797812991
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,1536,0.010230222509966956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,512,0.0037191112836201987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,256,0.0026764445420768526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,1024,0.009730666875839233
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,768,0.009775111244784461
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,128,0.0020951111283567217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,64,0.0020782223178280727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,512,0.00904266701804267
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,32,0.0020657777786254883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,256,0.009120889008045197
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,16384,0.009919111099508073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,128,0.008393777741326226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,64,0.008014221986134848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,65536,0.05347555544641283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,12288,0.006067555397748947
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,32,0.007928888830873702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,10240,0.005614222337802251
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,8192,0.006767999794748094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,16384,0.024392000503010217
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,65536,0.07481688923305936
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,12288,0.020602666669421725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,7168,0.00509333320789867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,6144,0.0046773333516385825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,5120,0.004622222234805425
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,10240,0.018610666195551556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,8192,0.016591111818949383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,4096,0.004589333302444882
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,7168,0.015613334046469795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,3584,0.016421332955360413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,6144,0.014615111880832247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,3072,0.014318222800890604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,5120,0.013792888985739814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,2560,0.012210666305489011
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,4096,0.012655999925401477
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,2048,0.010082667072614035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,3584,0.012250666817029318
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,3072,0.011733333269755045
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,2560,0.011160888605647616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,1024,0.0058408888677755995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,768,0.004770666774776247
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,2048,0.01071466671095954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,1536,0.00794311116139094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,512,0.0037164443896876443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,1536,0.010233778092596266
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,256,0.0026497778793176017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,128,0.0021155555215146807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,768,0.009672000176376766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,64,0.0020764445265134177
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,512,0.009111111362775167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,256,0.009461333354314169
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,32,0.002052444343765577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,1024,0.009773333039548662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,128,0.008447999755541483
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,65536,0.011727111207114326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,16384,0.006671111202902264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,12288,0.0057964444988303715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,64,0.007796444826655918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,32,0.007898666792445714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,10240,0.005331555588377847
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,8192,0.004915555732117759
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,16384,0.02443555494149526
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,7168,0.031122667921913996
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,12288,0.02050933407412635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,65536,0.07124800152248807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,10240,0.018618666463428073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,6144,0.0270506673389011
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,5120,0.02274133265018463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,8192,0.01660355594423082
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,4096,0.01851377718978458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,7168,0.015636444091796875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,3584,0.016417778200573392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,6144,0.01460444430510203
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,5120,0.013776889277829064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,3072,0.014311111635631986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,4096,0.012735111018021902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,2560,0.01219911128282547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,2048,0.010080888867378235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,3584,0.01219377749496036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,3072,0.011738667057620155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,1536,0.00795999997191959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,1024,0.0058222222659322954
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,768,0.004744889007674323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,2560,0.011162666810883416
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,2048,0.010712888505723743
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,1536,0.01017066670788659
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,512,0.003693333516518275
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,1024,0.009776888622177972
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,256,0.002625777696569761
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,768,0.009850666754775578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,128,0.0020924444413847392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,64,0.0020577777177095413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,512,0.008963555925422246
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,32,0.0020737778395414352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,256,0.009200889203283522
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,128,0.00851999968290329
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,65536,0.00924711094962226
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,16384,0.006578666468461354
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,64,0.007999999655617608
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,12288,0.005668444352017508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,32,0.007655999726719326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,10240,0.0052391112678580815
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,65536,0.07110844718085395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,8192,0.03551377852757772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,16384,0.02440266642305586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,12288,0.020489777127901714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,10240,0.018604444132910836
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,7168,0.031103110975689355
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,6144,0.02699377801683214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,8192,0.016612443659040663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,5120,0.022708444131745234
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,7168,0.015589333242840238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,4096,0.0184871107339859
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,6144,0.01460355520248413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,3584,0.016403555870056152
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,5120,0.013582222163677216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,3072,0.01428533262676663
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,4096,0.01275111072593265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,2560,0.012215110990736218
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,2048,0.010077333284748925
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,3584,0.012144889268610211
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,3072,0.01163822246922387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,1024,0.005821333163314396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,1536,0.007941332956155142
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,768,0.0047306666771570844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,2560,0.011092444260915121
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,2048,0.01073688848151101
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,512,0.0036880001425743103
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,1536,0.010222222242090438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,256,0.0026275554878844153
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,1024,0.009631111390060848
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,128,0.002096000024014049
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,64,0.002055111030737559
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,32,0.0020657777786254883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,768,0.009752888646390703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,512,0.009104889300134447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,256,0.009467555416954888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,128,0.008482666479216682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,64,0.007691555553012424
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,65536,0.008981333010726506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,16384,0.0064897777305708984
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,12288,0.005599110904667113
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,32,0.007575111256705389
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,10240,0.005097777893145879
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,16384,0.024159999357329473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,65536,0.07008355855941772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,8192,0.03542577889230516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,7168,0.03079644507831997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,12288,0.0203146669599745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,10240,0.018389332625601027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,6144,0.026939556002616882
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,5120,0.022480888499153986
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,8192,0.016318221886952717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,4096,0.018283555905024212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,7168,0.01555288831392924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,3584,0.016208888755904306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,6144,0.014608888162506951
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,3072,0.014129777749379476
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,5120,0.01348977784315745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,4096,0.012664000193277994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,2048,0.009951111343171861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,2560,0.01202044470442666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,3584,0.01202311118443807
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,1536,0.007915555602974361
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,1024,0.005764444255166584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,3072,0.011692444483439127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,2560,0.011221333510345884
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,2048,0.010587555666764578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,768,0.0047475554876857335
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,512,0.0036888888312710654
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,1536,0.010168888502650792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,256,0.0026213334252436957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,1024,0.009560888840092553
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,128,0.0020782223178280727
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,64,0.002054222135080232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,768,0.00963644435008367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,512,0.009213333328564962
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,32,0.002063111091653506
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,256,0.009364444348547194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,128,0.008354666332403818
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,64,0.007559110720952352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,32,0.007796444826655918
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,10240,1.162334230211046
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,12288,1.4052986568874781
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,16384,1.8416186438666449
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,10240,1.7927502526177301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,8192,0.8924488491482205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,12288,2.1518241034613714
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,8192,1.4339698155721028
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,7168,0.7531510988871256
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,7168,1.2547156016031902
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,6144,0.6191315650939941
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,5120,0.5040097766452366
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,16384,2.8675208621554904
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,6144,1.077198240492079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,4096,0.40361865361531574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,5120,0.8973528544108073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,3072,0.31186045540703666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,4096,0.7435813479953342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,3584,0.3882435427771674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,3584,0.6500337918599447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,2560,0.46289777755737305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,2048,0.3698391119639079
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,2048,0.21806044048733184
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,2560,0.2844853401184082
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,1536,0.27736711502075195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,1536,0.16706400447421602
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,1024,0.12048978275722927
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,3072,0.6311919954087999
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,768,0.14958133962419298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,128,0.025768000218603346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,256,0.04786577820777893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,64,0.0064328888224230866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,512,0.11048977904849583
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,32,0.004052444464630551
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,256,0.07839377721150716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,64,0.07937422063615587
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,768,0.10277955399619208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,1024,0.22354843881395128
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,512,0.08308888806237115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,32,0.08128355609046088
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,128,0.07895377609464857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,12288,0.3659164375729031
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,16384,0.7196106910705566
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,16384,0.47449779510498047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,10240,0.47057686911688906
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,10240,0.3033253351847331
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,12288,0.6106399959988064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,8192,0.22620977295769584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,8192,0.36040178934733075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,7168,0.20068800449371338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,6144,0.19077066580454508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,7168,0.3562720086839464
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,6144,0.3062320020463732
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,4096,0.1813946697447035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,5120,0.2264231046040853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,3072,0.1367759969499376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,5120,0.15922666920555964
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,4096,0.13298755221896702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,3584,0.11928089459737141
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,3584,0.17901955710517037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,3072,0.10445066955354478
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,2048,0.10682400067647298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,2560,0.08339555395974053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,65536,1.6546497344970703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,2048,0.06691910823186238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,1536,0.08040089077419706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,768,0.036883556180530123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,1536,0.05215555429458618
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,1024,0.054515557156668774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,2560,0.12821155124240452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,1024,0.030216889248953924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,256,0.015274667077594332
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,512,0.012248000337017907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,128,0.003169777699642711
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,64,0.0042160000238153665
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,768,0.013644444445768992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,32,0.0024293334119849736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,128,0.009840889109505547
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,64,0.009206222163306342
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,256,0.011090666883521609
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,512,0.012240000069141388
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,32,0.009263111485375298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,12288,0.27554933230082196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,10240,0.3386791017320421
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,16384,0.32591377364264595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,16384,0.5405528810289171
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,12288,0.45949957105848527
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,65536,2.8874969482421875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,8192,0.2708684338463677
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,10240,0.22776532173156738
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,8192,0.17113155788845488
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,7168,0.23772978782653809
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,7168,0.15464088651869032
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,5120,0.19224622514512804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,4096,0.13670489523145887
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,6144,0.23036623001098633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,6144,0.14111733436584473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,3584,0.1348151101006402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,3072,0.11605511771308051
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,5120,0.12073777781592475
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,2560,0.08603911267386542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,65536,1.2140159606933594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,4096,0.10063288609186809
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,3072,0.07366310887866549
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,2048,0.07193422317504883
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,2560,0.06227911180920071
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,1024,0.03700444433424208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,3584,0.08803200059466892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,2048,0.046590222252739795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,768,0.031183110343085393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,1536,0.060975107881757945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,512,0.006856888946559694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,1024,0.0125608891248703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,1536,0.039577778842714094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,256,0.01111644423670239
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,768,0.01239111108912362
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,64,0.003626666549179289
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,128,0.006395555618736479
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,32,0.0023786665664778813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,128,0.009139555195967356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,256,0.010455999937322406
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,32,0.008991111483838823
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,64,0.00909511082702213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,512,0.011408000356621213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,65536,2.1693572998046875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,12288,0.2116168869866265
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,16384,0.30060267448425293
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,12288,0.34072444174024796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,16384,0.4511724578009711
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,10240,0.28289956516689724
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,8192,0.14180089367760554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,10240,0.17496711677975127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,8192,0.25519733958774143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,7168,0.19861866368187797
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,7168,0.1243528922398885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,65536,1.0130728615654838
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,6144,0.10971555444929336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,5120,0.09451378054089016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,5120,0.16059999995761448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,3584,0.1002533303366767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,6144,0.1924097802903917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,4096,0.0814631117714776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,2560,0.07530044184790717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,3072,0.09711644384596084
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,4096,0.12862222724490696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,3584,0.07121421893437703
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,2048,0.060511112213134766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,1536,0.05130489004982842
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,2560,0.053074667851130165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,3072,0.05721600188149346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,1536,0.02108888824780782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,1024,0.03125066558519999
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,768,0.006766222417354584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,256,0.01033688916100396
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,768,0.011786667009194693
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,1024,0.012136889000733694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,2048,0.04411555661095513
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,512,0.005296000176005893
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,128,0.005054222212897407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,65536,1.8070941501193578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,64,0.003526222374704149
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,32,0.0023111111174027124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,256,0.009699555734793345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,512,0.01128000020980835
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,64,0.008928000099129146
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,32,0.008823110825485652
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,128,0.00923644420173433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,12288,0.16611644956800672
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,16384,0.21618489424387613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,10240,0.226785765753852
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,16384,0.3614693217807346
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,10240,0.15137688318888345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,12288,0.2993653350406223
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,8192,0.1169182194603814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,7168,0.10444444417953491
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,6144,0.0906568898095025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,8192,0.1819279988606771
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,6144,0.15451378292507595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,5120,0.07455999983681573
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,65536,0.7780515352884928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,7168,0.17965955204433867
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,5120,0.11474133862389459
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,4096,0.09193422396977742
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,4096,0.05937866369883219
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,3072,0.07305955224566989
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,3584,0.05266933308707344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,2560,0.06087377998563978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,2048,0.049061334795422025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,3072,0.04960088928540548
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,2560,0.039629333549075656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,1536,0.03717866539955139
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,768,0.006046222315894232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,2048,0.029110223054885864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,1024,0.0254737784465154
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,1536,0.01311644415060679
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,512,0.004805333498451444
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,768,0.011657777759763928
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,3584,0.09072355429331462
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,128,0.004318222403526306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,1024,0.012154666913880242
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,64,0.002959111084540685
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,128,0.009161777794361115
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,64,0.009138666921191746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,256,0.010230222509966956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,32,0.0022026666750510535
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,65536,1.4467119640774195
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,32,0.009098666409651438
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,12288,0.23967112435234916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,256,0.00332622230052948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,512,0.011319999893506369
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,16384,0.357223113377889
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,12288,0.14162666267818874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,16384,0.2035315566592746
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,65536,0.6762960222032336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,10240,0.2242435614267985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,10240,0.13065332836574978
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,8192,0.10071911414464314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,7168,0.1576568947898017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,8192,0.17954933643341064
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,5120,0.10066044330596924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,7168,0.08945244550704956
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,6144,0.135535995165507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,6144,0.08158666557735868
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,5120,0.06731377707587348
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,4096,0.09073333607779609
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,4096,0.05093600021468269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,3072,0.07233777973386976
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,3584,0.07968533039093018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,65536,1.2671057383219402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,3072,0.04375288883845011
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,2560,0.053823110130098134
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,2560,0.03508177730772231
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,2048,0.04857600066396925
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,1024,0.011208000282446543
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,1536,0.012574222352769641
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,2048,0.017215111189418368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,1536,0.03676533367898729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,3584,0.04976444443066915
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,768,0.0058666666348775225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,512,0.0047191112405723995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,256,0.0063440000845326324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,768,0.011310222248236338
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,128,0.003919110943873723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,1024,0.02387822171052297
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,64,0.0027946666296985415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,256,0.009871111147933537
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,512,0.011071999867757162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,128,0.009282666775915358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,64,0.008849778109126622
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,32,0.002181333386235767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,32,0.009091555244392818
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,12288,0.20627378092871773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,16384,0.16085510783725315
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,16384,0.27186134126451283
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,10240,0.1711182196935018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,12288,0.13331822554270426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,8192,0.13707644409603542
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,10240,0.10476177930831909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,8192,0.08647021982404922
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,7168,0.1204942199918959
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,6144,0.10364266898896958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,5120,0.08676799800660874
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,6144,0.0640960004594591
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,4096,0.06954577896330091
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,7168,0.08091200060314603
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,4096,0.04458933406405979
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,3584,0.06113955709669325
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,65536,0.6456551021999782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,3072,0.055495109823015004
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,3584,0.04021777709325155
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,2560,0.04651022288534376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,1536,0.028714666763941448
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,3072,0.036497778362698026
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,1024,0.019697777099079557
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,1536,0.01201600001917945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,2048,0.04191644324196709
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,768,0.005690666536490123
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,1024,0.011018666956159802
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,5120,0.0590462221039666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,512,0.004926222066084544
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,2560,0.020415110720528495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,256,0.004973333328962326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,768,0.011327111058764987
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,2048,0.012402666939629449
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,64,0.0026737778551048706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,512,0.010579555398888059
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,128,0.00350577798154619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,65536,1.085911962721083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,32,0.0021395554973019496
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,256,0.009741333623727163
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,128,0.008967999782827165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,64,0.008884444832801819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,32,0.00868977771864997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,10240,0.14300178156958684
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,16384,0.133023116323683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,12288,0.10382844342125787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,16384,0.22744266192118326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,10240,0.08826844559775458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,12288,0.17271110746595594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,8192,0.11461155944400364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,6144,0.09118488762113784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,6144,0.054430223173565336
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,5120,0.07271555397245619
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,7168,0.10087377495235866
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,65536,0.4840879970126682
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,8192,0.07058755556742351
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,4096,0.05837511354022556
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,3584,0.054492443799972534
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,5120,0.04595288965437147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,7168,0.06748000118467543
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,4096,0.03849688834614224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,3584,0.035122666094038225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,2048,0.031847112708621554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,65536,0.9085395601060656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,2560,0.012823999755912356
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,2560,0.03966933488845825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,1536,0.024528889192475214
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,2048,0.012081777883900536
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,3072,0.02068711154990726
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,1536,0.011563555234008364
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,1024,0.018748443987634447
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,1024,0.010810666614108615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,3072,0.05274488859706455
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,512,0.004831111265553368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,256,0.004399111287461387
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,768,0.011041777829329172
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,768,0.005442666510740916
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,512,0.010255999863147736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,128,0.00263288886182838
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,256,0.009351111120647853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,32,0.0021226666867733
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,64,0.002529777793420686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,64,0.008714666797055138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,128,0.008884444832801819
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,32,0.008654221892356873
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,16384,0.18284355269538033
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,16384,0.1182675494088067
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,12288,0.13952267169952393
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,12288,0.08471377690633138
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,10240,0.1293920013639662
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,8192,0.09239821963840061
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,7168,0.08123644193013509
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,65536,0.38917332225375706
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,6144,0.07004088825649686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,8192,0.05734399954477946
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,7168,0.05081511206097073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,10240,0.07209689087337919
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,5120,0.05877066983116997
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,6144,0.044213334719340004
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,4096,0.050103111399544605
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,3584,0.0444453325536516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,5120,0.03845422135459052
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,4096,0.02446577780776554
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,3584,0.01540177729394701
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,65536,0.7260079913669162
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,2048,0.02590844366285536
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,2560,0.03233511249224345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,3072,0.012432888978057437
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,1536,0.007986666427718269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,2560,0.012176888684431711
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,3072,0.03836266530884637
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,1024,0.014044443766276041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,768,0.00510044437315729
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,512,0.004068444586462445
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,1536,0.011046222514576383
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,768,0.010441777606805166
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,128,0.00286666676402092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,2048,0.011296889020336999
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,256,0.0038355555799272326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,1024,0.010419555836253695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,64,0.0024008889579110676
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,256,0.008972444468074376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,512,0.010244444840484196
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,128,0.008170666793982187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,32,0.0020471111767821843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,32,0.008223111430803934
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,64,0.007986666427718269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,16384,0.09617066383361816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,16384,0.16035377979278564
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,12288,0.13762844933403864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,10240,0.10108977556228638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,12288,0.07400621970494588
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,8192,0.0811377763748169
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,10240,0.062365333239237465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,65536,0.34183287620544434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,7168,0.07160177495744494
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,8192,0.050445334778891675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,5120,0.05174755718972948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,6144,0.06167111131880018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,7168,0.045048889186647206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,4096,0.04462044437726339
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,6144,0.039805332819620766
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,5120,0.03334044416745504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,3584,0.039847112364239164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,65536,0.6379111078050401
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,3072,0.034161776304244995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,2560,0.029095109966066148
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,4096,0.013436444103717804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,3072,0.012240889171759287
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,1536,0.018142221702469718
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,1024,0.012478222449620565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,2560,0.011947555674446953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,768,0.005045333256324132
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,1536,0.010899555351999072
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,512,0.006585777633719974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,1024,0.010219555762079027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,256,0.003653333418899112
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,2048,0.023527999718983967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,3584,0.013105777402718862
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,512,0.00981155534585317
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,128,0.0023599999646345773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,768,0.010276444256305695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,64,0.0023511110080613033
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,256,0.009109333157539368
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,32,0.0020471111767821843
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,128,0.008278222547637092
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,64,0.007900444169839224
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,32,0.007891555627187094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,16384,0.1375866731007894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,12288,0.10548622078365749
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,2048,0.011318222516112857
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,12288,0.06598222255706787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,10240,0.08693333466847737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,8192,0.06976177957322863
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,65536,0.2958453231387668
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,10240,0.05426311161783007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,16384,0.08346222506629096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,7168,0.061680886480543345
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,8192,0.043980442815356784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,5120,0.047320889102088086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,65536,0.5444213549296061
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,6144,0.03533777925703261
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,7168,0.03975644376542833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,6144,0.05518044365776909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,5120,0.015959110524919298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,4096,0.037788444095187716
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,3584,0.033732444047927856
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,3072,0.02918044394916958
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,3584,0.012827555338541666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,4096,0.013048888908492194
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,3072,0.012144000165992312
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,1536,0.015722667177518208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,2560,0.024772443705134924
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,1024,0.0052639999323421055
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,768,0.004980444494220945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,512,0.005383111122581694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,1024,0.009910222556855943
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,768,0.010411555568377176
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,2048,0.022153778208626643
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,1536,0.01091822236776352
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,256,0.003933333274390963
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,2048,0.011146666275130378
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,64,0.0023235556566052968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,512,0.00979200005531311
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,128,0.0022764443937275144
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,256,0.008952889177534316
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,32,0.0020826665891541377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,128,0.008072000410821702
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,64,0.007998222278224098
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,2560,0.011882666912343768
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,32,0.007826666865083907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,16384,0.1162373357348972
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,10240,0.07372977998521593
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,12288,0.08968889051013523
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,16384,0.07024888859854804
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,12288,0.05576711230807834
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,8192,0.05915021896362305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,65536,0.24945510758293998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,10240,0.04559466573927137
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,7168,0.05231022172504001
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,6144,0.0476746658484141
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,7168,0.03203644355138143
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,5120,0.039432889885372586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,8192,0.03830666674507989
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,4096,0.03168888886769613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,3584,0.02815999918513828
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,6144,0.01568000018596649
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,5120,0.013979555832015144
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,3072,0.024548444483015273
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,65536,0.45332887437608504
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,4096,0.012936000194814471
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,2560,0.020779555042584736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,2048,0.0075146663520071245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,3584,0.012749333348539142
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,1536,0.013117333253224691
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,1024,0.004834666848182678
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,3072,0.011918221910794577
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,2560,0.011776000261306763
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,768,0.0063884444534778595
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,512,0.005548444473081165
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,2048,0.011065777805116443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,1536,0.010774222513039907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,256,0.003050666716363695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,1024,0.010008888940016428
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,768,0.010059555371602377
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,64,0.002179555594921112
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,32,0.0020497778637541667
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,256,0.008467555873923832
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,128,0.008086221913496653
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,64,0.007970666719807519
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,512,0.009696000152164036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,128,0.002199999988079071
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,32,0.007754666937722101
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,12288,0.07258933120303683
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,16384,0.09323288997014363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,10240,0.05908888578414917
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,16384,0.05747289127773709
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,8192,0.047777778572506376
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,7168,0.042252444558673434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,65536,0.2049706644482083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,12288,0.050817777713139854
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,8192,0.020340444313155282
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,6144,0.03716444306903415
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,10240,0.03834755553139581
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,65536,0.3636559910244412
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,5120,0.031179553932613794
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,6144,0.01475644442770216
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,4096,0.02529955572552151
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,5120,0.013819555441538492
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,3072,0.019712888532214694
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,4096,0.012897777888509961
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,2560,0.016736888223224215
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,3584,0.025007999605602686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,3072,0.011894221935007306
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,7168,0.015978667471143935
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,2048,0.013703111145231458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,3584,0.012729778057999082
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,1536,0.006020444548792309
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,2560,0.011575111084514193
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,1024,0.007965332931942409
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,2048,0.010802666346232096
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,768,0.004655111167165968
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,512,0.004609777695602841
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,1536,0.010688888529936472
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,256,0.003061333257291052
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,1024,0.009947555760542551
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,128,0.002228444442152977
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,768,0.010024888647927178
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,512,0.009810667071077559
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,64,0.0022675556441148124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,32,0.002032888846264945
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,256,0.008842666943868002
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,128,0.008083555433485243
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,64,0.007963555554548899
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,32,0.00776444458299213
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,16384,0.07509422302246094
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,12288,0.05983466572231717
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,10240,0.048044443130493164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,8192,0.0381644434399075
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,16384,0.04662933283381992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,65536,0.15655822224087185
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,10240,0.019184889064894784
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,12288,0.03698577814631992
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,7168,0.03477422065205044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,65536,0.2894017696380615
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,8192,0.0165928883685006
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,6144,0.026054221722814772
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,5120,0.010646222366227044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,7168,0.01575288838810391
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,6144,0.014684443672498068
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,3584,0.017289777596791584
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,5120,0.013796444568369122
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,3072,0.015146666102939181
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,4096,0.012736000120639801
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,2560,0.012880889077981314
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,3584,0.012628444366984896
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,4096,0.02159022291501363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,3072,0.011760888828171624
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,1536,0.00849333322710461
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,2560,0.01162755572133594
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,2048,0.00583022211988767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,1024,0.0030177777840031516
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,2048,0.010842666857772403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,1536,0.0107004443804423
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,512,0.0032391111469931076
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,256,0.002700444517864121
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,768,0.00396799999806616
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,1024,0.00965244405799442
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,128,0.002152888932161861
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,768,0.010088889135254754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,64,0.002157333410448498
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,32,0.002040000011523565
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,512,0.00962311112218433
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,256,0.008736000292830998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,128,0.007882666256692674
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,64,0.007762666377756331
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,32,0.007954667011896769
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,16384,0.05156088868776957
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,12288,0.04057333204481337
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,10240,0.033321777979532875
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,16384,0.02827733423974779
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,8192,0.013730666703648038
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,12288,0.02049511174360911
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,65536,0.1946106619305081
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,10240,0.018571555614471436
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,6144,0.008945778012275696
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,7168,0.01000355515215132
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,8192,0.016536000702116225
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,7168,0.01565511193540361
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,5120,0.016599110431141324
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,4096,0.007880888879299164
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,65536,0.1140257782406277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,3584,0.012287110918098025
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,5120,0.01366133325629764
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,3072,0.010978666444619497
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,4096,0.012664889295895895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,2560,0.009535111486911774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,6144,0.014688000082969666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,3584,0.012631999949614206
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,2048,0.005364444520738389
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,1536,0.00600622221827507
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,3072,0.011759111450778114
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,1024,0.0028773333049482773
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,768,0.003740444365474913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,2560,0.011632888681358762
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,2048,0.010655110908879174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,1536,0.010588444769382477
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,512,0.0028782222006056043
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,256,0.0025964443468385274
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,1024,0.009582222335868413
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,768,0.010053333308961656
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,128,0.002025777681006326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,64,0.0020666666742828158
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,32,0.002025777681006326
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,256,0.008676444490750631
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,512,0.009728888670603434
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,128,0.007752888732486301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,64,0.008000888758235509
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,32,0.007690666450394525
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,16384,0.03707733419206407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,12288,0.03094044327735901
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,65536,0.13767288790808782
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,16384,0.024200000696712073
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,8192,0.009170666337013245
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,10240,0.008928889201747047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,65536,0.09954310788048638
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,12288,0.020484444167878892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,7168,0.008590222232871586
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,10240,0.018568888306617737
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,6144,0.0075804442167282104
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,5120,0.013224889006879596
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,8192,0.01647200021478865
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,4096,0.011519999967681037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,3584,0.009759999811649323
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,6144,0.014634667171372307
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,3072,0.009008888569143083
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,5120,0.013702222042613559
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,2560,0.003679111186001036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,7168,0.015621332658661736
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,2048,0.0031351111829280853
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,4096,0.012643555800120035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,3584,0.01239377756913503
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,1536,0.003050666716363695
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,1024,0.002752888947725296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,3072,0.011748444702890186
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,768,0.0028986665937635633
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,2048,0.010661332971519895
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,512,0.002667555585503578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,1536,0.01054933336046007
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,1024,0.009535111486911774
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,256,0.0025635556214385563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,2560,0.011535111400816174
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,128,0.0019724443554878235
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,768,0.009937778115272522
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,64,0.001965333397189776
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,512,0.009709333380063375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,32,0.0020346666375796
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,256,0.008544888761308458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,128,0.007777777810891469
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,64,0.007897777689827813
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,16384,0.025871111287011042
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,32,0.00792711145348019
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,12288,0.010889777706729041
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,10240,0.016559999850061204
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,65536,0.09346577856275771
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,8192,0.007159999675220913
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,16384,0.02424000038041009
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,7168,0.0070471109615431885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,12288,0.02048000031047397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,6144,0.006015111174848344
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,65536,0.08850755294164021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,10240,0.0184871107339859
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,8192,0.016543999314308167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,5120,0.008752889103359645
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,4096,0.00907733374171787
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,3584,0.004049777984619141
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,7168,0.0155102229780621
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,3072,0.006425777657164468
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,5120,0.013677332964208392
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,2560,0.003552888830502828
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,4096,0.01257511145538754
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,3584,0.012483555409643384
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,6144,0.01458666721979777
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,2048,0.0047306666771570844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,1536,0.0029484445436133277
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,3072,0.011775111158688864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,1024,0.0027466666781240036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,768,0.0027742222365405825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,512,0.002529777793420686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,2048,0.010568000376224518
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,1536,0.0105591110057301
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,2560,0.011571555501884885
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,1024,0.009576000273227692
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,768,0.010068444742096795
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,256,0.0022960000981887183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,128,0.0019111111760139465
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,64,0.0018382221460342407
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,512,0.009429333110650381
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,32,0.002020444514022933
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,256,0.008619555168681675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,128,0.007835555407736037
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,64,0.007701333198282454
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,65536,0.0484186675813463
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,32,0.00795733349190818
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,12288,0.006249777972698212
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,16384,0.006792000184456508
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,16384,0.02422488894727495
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,10240,0.005256888767083486
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,65536,0.07492888636059232
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,8192,0.00489777781897121
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,12288,0.020423110988405015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,10240,0.018439110782411363
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,7168,0.008979555633332994
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,6144,0.012341332932313284
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,5120,0.003953777667548922
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,4096,0.0052426668504873914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,8192,0.01650399963061015
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,7168,0.015635554989178974
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,5120,0.01370044466522005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,3584,0.003383999897374047
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,6144,0.014601776997248331
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,3072,0.0031386667655573953
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,4096,0.012533333566453723
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,2560,0.0028808888875775864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,3584,0.012452444268597497
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,2048,0.002521777732504739
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,1536,0.0024844445288181305
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,3072,0.011785777906576792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,2560,0.011436444189813403
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,1024,0.0022239999638663395
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,768,0.0023653333385785422
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,2048,0.01055466632048289
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,1536,0.010460444622569613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,512,0.0022479999396536085
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,256,0.002044444489810202
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,1024,0.009597332941161262
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,128,0.00175111119945844
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,64,0.0017413333472278381
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,512,0.009641778137948778
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,768,0.009863110880057016
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,32,0.002004444392191039
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,256,0.0083155557513237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,128,0.007989333735571967
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,16384,0.005016889009210798
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,65536,0.01658933361371358
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,64,0.007655111451943715
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,32,0.00794222205877304
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,12288,0.005417777846256892
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,16384,0.024149333437283833
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,8192,0.006436444405052397
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,10240,0.0047191112405723995
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,65536,0.07074133555094402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,12288,0.02040799955526988
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,7168,0.0063768890168931745
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,10240,0.01846222248342302
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,6144,0.006331555545330048
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,5120,0.0036480000449551474
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,4096,0.0030497778207063675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,8192,0.016523554921150208
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,7168,0.015653333730167814
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,6144,0.01457244488928053
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,3584,0.003083555648724238
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,5120,0.013535999589496188
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,3072,0.0028373334142896864
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,2560,0.0026453334010309647
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,4096,0.012643555800120035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,2048,0.0022168887986077205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,1536,0.0022675556441148124
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,3584,0.012337778177526263
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,3072,0.011763555308183035
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,1024,0.001998222122589747
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,2560,0.01144177797767851
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,768,0.0022444443570242985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,2048,0.010716444088353051
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,256,0.001975111042459806
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,512,0.002181333386235767
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,1536,0.010587555666764578
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,1024,0.009591999981138442
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,768,0.010055111514197456
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,128,0.0016986666661169792
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,64,0.001671111107700401
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,512,0.00944266633854972
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,32,0.001992000059949027
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,256,0.008630221916569604
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,128,0.007654222349325816
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,65536,0.0075288886825243635
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,16384,0.0052577778697013855
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,64,0.00777333312564426
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,32,0.007552000383536021
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,12288,0.004679111142953237
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,10240,0.004492444296677907
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,16384,0.024196444286240473
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,8192,0.005140444470776452
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,65536,0.07075288560655382
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,12288,0.020463110672103036
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,10240,0.018383999665578205
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,7168,0.005024000174469418
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,6144,0.005104889058404499
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,8192,0.01637066735161675
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,5120,0.004300444490379757
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,4096,0.0029333333174387612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,7168,0.015528000063366361
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,3584,0.0029848888516426086
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,3072,0.002757333426011933
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,5120,0.013647110925780402
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,6144,0.014706666270891825
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,4096,0.012521777715947894
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,2560,0.002605333303411802
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,2048,0.0021768889079491296
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,3584,0.012353777885437012
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,1536,0.002208000048995018
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,3072,0.011584888729784222
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,1024,0.0019457776927285725
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,2560,0.01145244472556644
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,768,0.0022151112142536375
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,2048,0.010495999621020423
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,512,0.0021173333128293357
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,1536,0.010510221951537661
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,256,0.0019466665883858998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,1024,0.009509333305888707
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,128,0.001681777752108044
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,768,0.009798222117953831
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,512,0.009434666898515489
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,64,0.001689777813023991
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,32,0.0019377778387731975
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,256,0.008657777474986183
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,65536,0.00508266646001074
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,128,0.007627555893527136
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,64,0.00792266676823298
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,16384,0.003721777763631609
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,12288,0.003535110917356279
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,32,0.007790221936172909
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,10240,0.0035333335399627686
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,8192,0.003932444585694207
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,65536,0.06954310999976264
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,16384,0.023874666955735948
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,7168,0.004013333469629288
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,12288,0.020202666521072388
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,6144,0.003970666478077571
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,8192,0.016311999824311998
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,5120,0.0034942221310403612
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,10240,0.018364444375038147
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,4096,0.002903999967707528
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,3584,0.002971555623743269
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,7168,0.015295111470752291
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,6144,0.01441511180665758
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,3072,0.002716444432735443
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,5120,0.013577777478430005
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,2560,0.002560888934466574
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,4096,0.012460444536474017
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,2048,0.002275555498070187
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,3584,0.012318222059143914
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,1536,0.002161777681774563
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,3072,0.01165066659450531
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,1024,0.0019173332386546666
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,768,0.002179555594921112
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,2560,0.011321778098742167
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,2048,0.010703999963071613
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,512,0.0020853332761261198
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,256,0.0019484443797005548
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,1536,0.01036800030204985
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,1024,0.009679111341635386
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,128,0.0016542221936914658
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,64,0.0016622222546074127
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,768,0.009966221948464712
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,32,0.0018355554590622585
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,512,0.009249777429633671
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,128,0.007635555333561367
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,256,0.008730666504965888
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,64,0.007807999849319458
VLLM,0.12.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,32,0.007781333393520779
