framework,version,device,op_name,kernel_source,m,k,quant_dtype,latency
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,16384,fp8,0.36347834269205737
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,12288,fp8,0.25387948354085277
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,10240,fp8,0.19825388590494797
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,8192,fp8,0.13369280497233071
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,7168,fp8,0.09663146336873374
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,6144,fp8,0.05311147371927899
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,5120,fp8,0.0
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,4096,fp8,0.018909867604573583
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,3584,fp8,0.03822933435440064
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,3072,fp8,0.03099306424458821
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,2560,fp8,0.02662400205930074
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,2048,fp8,0.02239253322283427
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,1536,fp8,0.01836373209953308
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,1024,fp8,0.014609066645304364
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,768,fp8,0.012868266304334006
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,512,fp8,0.010786132514476776
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,256,fp8,0.009352533022562663
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,128,fp8,0.00815786694486936
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,64,fp8,0.007475199798742931
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,32,fp8,0.007305599997440974
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8192,65536,fp8,1.6088404337565105
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,16384,fp8,0.12963841756184896
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,12288,fp8,0.05116585095723475
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,10240,fp8,0.0
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,8192,fp8,0.021674672762552902
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,7168,fp8,0.03724160194396972
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,6144,fp8,0.031191464265187582
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,5120,fp8,0.026487467686335246
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,4096,fp8,0.022089600563049313
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,3584,fp8,0.02024106582005819
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,3072,fp8,0.018364801009496053
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,2560,fp8,0.016452267765998837
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,2048,fp8,0.014574933052062991
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,1536,fp8,0.0127701332171758
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,1024,fp8,0.010794665912787119
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,768,fp8,0.00980373372634252
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,512,fp8,0.009447466333707173
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,256,fp8,0.007850666344165803
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,128,fp8,0.007507200042406718
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,64,fp8,0.007372800260782241
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,32,fp8,0.00716373287141323
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4096,65536,fp8,0.7898719787597659
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,16384,fp8,0.04037973483403523
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,12288,fp8,0.030812799930572524
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,10240,fp8,0.026416001717249553
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,8192,fp8,0.02228373289108277
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,7168,fp8,0.02038506468137105
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,6144,fp8,0.018292266130447387
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,5120,fp8,0.016515199343363447
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,4096,fp8,0.01467733383178711
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,3584,fp8,0.01389226714769999
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,3072,fp8,0.013374933600425722
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,2560,fp8,0.011776000261306764
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,2048,fp8,0.010755200684070588
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,1536,fp8,0.009864532947540283
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,1024,fp8,0.008738133311271667
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,768,fp8,0.008396800359090168
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,512,fp8,0.008049066613117852
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,256,fp8,0.007518933465083439
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,128,fp8,0.007277866328756015
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,64,fp8,0.007202133163809775
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,32,fp8,0.007210667058825493
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2048,65536,fp8,0.35894184112548844
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,16384,fp8,0.02232639988263448
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,12288,fp8,0.01839786569277446
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,10240,fp8,0.016661333044370015
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,8192,fp8,0.014711467425028485
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,7168,fp8,0.013648001352945962
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,6144,fp8,0.012663466731707256
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,5120,fp8,0.011776000261306764
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,4096,fp8,0.010986665884653728
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,3584,fp8,0.01027413308620453
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,3072,fp8,0.009959466258684796
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,2560,fp8,0.009386666119098663
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,2048,fp8,0.008806399504343669
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,1536,fp8,0.008356266965468725
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,1024,fp8,0.00815786694486936
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,768,fp8,0.0076458672682444264
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,512,fp8,0.007714133709669113
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,256,fp8,0.007230933258930843
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,128,fp8,0.007168000315626462
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,64,fp8,0.007270399853587149
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,32,fp8,0.007168000067273776
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1024,65536,fp8,0.13062721888224282
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,16384,fp8,0.01843199928601583
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,12288,fp8,0.01566933393478394
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,10240,fp8,0.014301867286364239
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,8192,fp8,0.01262933313846588
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,7168,fp8,0.012074666221936543
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,6144,fp8,0.011946667234102886
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,5120,fp8,0.010613333682219185
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,4096,fp8,0.009898666044076283
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,3584,fp8,0.009325866897900898
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,3072,fp8,0.009011200070381165
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,2560,fp8,0.00890666643778483
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,2048,fp8,0.008294400324424108
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,1536,fp8,0.007987199972073238
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,1024,fp8,0.007783466577529907
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,768,fp8,0.007543466488520305
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,512,fp8,0.007372800260782241
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,256,fp8,0.007209599887331325
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,128,fp8,0.007202132915457091
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,64,fp8,0.007106133426229159
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,32,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,768,65536,fp8,0.049493328730265296
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,16384,fp8,0.014745599031448366
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,12288,fp8,0.012793600559234621
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,10240,fp8,0.011878399054209393
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,8192,fp8,0.010887465874354045
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,7168,fp8,0.010480000575383504
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,6144,fp8,0.009966933230559031
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,5120,fp8,0.00938666711250941
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,4096,fp8,0.008941865960756938
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,3584,fp8,0.008772266159454983
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,3072,fp8,0.008460799604654312
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,2560,fp8,0.008227200309435526
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,2048,fp8,0.007953067123889924
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,1536,fp8,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,1024,fp8,0.007468799750010173
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,768,fp8,0.007505066941181819
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,512,fp8,0.007244800155361493
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,256,fp8,0.007202132915457091
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,128,fp8,0.007163733616471291
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,64,fp8,0.007162666693329811
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,32,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,512,65536,fp8,0.0477183977762858
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,16384,fp8,0.01279999911785126
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,12288,fp8,0.011332266529401145
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,10240,fp8,0.010615466038386026
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,8192,fp8,0.00993279963731766
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,7168,fp8,0.010069333016872408
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,6144,fp8,0.009113600353399913
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,5120,fp8,0.008874666442473731
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,4096,fp8,0.008294399827718735
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,3584,fp8,0.00812373384833336
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,3072,fp8,0.007987199972073238
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,2560,fp8,0.007652266820271811
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,2048,fp8,0.007611733675003052
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,1536,fp8,0.0074410667022069285
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,1024,fp8,0.007543466240167618
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,768,fp8,0.007240533704559009
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,512,fp8,0.007236266508698463
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,256,fp8,0.007236266508698463
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,128,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,64,fp8,0.007094399506847065
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,32,fp8,0.007098666951060295
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,384,65536,fp8,0.03136853377024333
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,16384,fp8,0.01085439920425415
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,12288,fp8,0.00993279963731766
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,10240,fp8,0.009454932808876038
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,8192,fp8,0.008878933389981587
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,7168,fp8,0.008673066645860674
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,6144,fp8,0.008362666765848795
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,5120,fp8,0.008131200075149534
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,4096,fp8,0.00815466692050298
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,3584,fp8,0.007748266309499741
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,3072,fp8,0.007618133475383123
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,2560,fp8,0.007509333391984304
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,2048,fp8,0.007508266220490138
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,1536,fp8,0.0075712002813816065
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,1024,fp8,0.007309866324067116
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,768,fp8,0.007338666295011839
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,512,fp8,0.007202132915457091
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,256,fp8,0.007133866722385089
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,128,fp8,0.007168000067273776
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,64,fp8,0.007168000067273776
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,32,fp8,0.007065599535902341
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,256,65536,fp8,0.02222079833348592
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,16384,fp8,0.009975467125574745
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,12288,fp8,0.009147733449935913
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,10240,fp8,0.008731733510891595
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,8192,fp8,0.008362666765848795
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,7168,fp8,0.008252799759308498
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,6144,fp8,0.008258133133252462
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,5120,fp8,0.007944533477226894
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,4096,fp8,0.007581866780916849
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,3584,fp8,0.007646933446327845
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,3072,fp8,0.007611733675003052
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,2560,fp8,0.007540266712506611
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,2048,fp8,0.007441066453854243
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,1536,fp8,0.007304533322652181
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,1024,fp8,0.0073727998882532125
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,768,fp8,0.006963199873765308
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,512,fp8,0.007270399853587149
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,256,fp8,0.007171200339992841
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,128,fp8,0.007168000315626462
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,64,fp8,0.007160532847046853
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,32,fp8,0.007139200096329053
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,192,65536,fp8,0.018397865692774452
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,16384,fp8,0.009560532867908478
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,12288,fp8,0.008738133062918981
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,10240,fp8,0.00849919964869817
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,8192,fp8,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,7168,fp8,0.007979733496904375
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,6144,fp8,0.007882666587829589
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,5120,fp8,0.0077824001510938025
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,4096,fp8,0.0077141329646110545
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,3584,fp8,0.007441066453854244
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,3072,fp8,0.007473066449165344
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,2560,fp8,0.007478400195638339
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,2048,fp8,0.0075807999819517136
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,1536,fp8,0.007304533198475838
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,1024,fp8,0.007202133163809775
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,768,fp8,0.007198933139443397
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,512,fp8,0.007236266508698463
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,256,fp8,0.007202132915457091
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,128,fp8,0.00715839999417464
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,64,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,32,fp8,0.007031466687719028
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,160,65536,fp8,0.016661334037780764
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,16384,fp8,0.009045333663622538
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,12288,fp8,0.008401066313187282
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,10240,fp8,0.008226133386294047
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,8192,fp8,0.007918933530648549
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,7168,fp8,0.0078165332476298
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,6144,fp8,0.007577600330114364
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,5120,fp8,0.007748266309499741
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,4096,fp8,0.007414400080839792
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,3584,fp8,0.00730666716893514
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,3072,fp8,0.007748266806205112
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,2560,fp8,0.007441066702206929
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,2048,fp8,0.007406933854023616
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,1536,fp8,0.007304533695181212
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,1024,fp8,0.007304533695181212
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,768,fp8,0.0072042662650346765
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,512,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,256,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,128,fp8,0.007099732880791028
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,64,fp8,0.007103999828298886
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,32,fp8,0.007065599535902341
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,128,65536,fp8,0.0146506667137146
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,16384,fp8,0.008396800359090168
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,12288,fp8,0.00808426613608996
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,10240,fp8,0.007714132964611054
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,8192,fp8,0.0075776000817616785
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,7168,fp8,0.007577599585056305
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,6144,fp8,0.007542399565378826
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,5120,fp8,0.007406933108965556
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,4096,fp8,0.007441066702206929
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,3584,fp8,0.007372800509134929
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,3072,fp8,0.007398400455713271
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,2560,fp8,0.00737813301384449
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,2048,fp8,0.0072277331103881215
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,1536,fp8,0.007168000315626462
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,1024,fp8,0.007165866966048876
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,768,fp8,0.007195733363429705
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,512,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,256,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,128,fp8,0.007102933153510093
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,64,fp8,0.0071295997748772304
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,32,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,96,65536,fp8,0.012868266304334006
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,16384,fp8,0.008192000289758047
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,12288,fp8,0.007816533495982488
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,10240,fp8,0.007748266309499739
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,8192,fp8,0.007438933104276656
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,7168,fp8,0.007584000130494436
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,6144,fp8,0.007468800495068233
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,5120,fp8,0.007406933854023616
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,4096,fp8,0.00734080026547114
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,3584,fp8,0.007364266862471898
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,3072,fp8,0.007304533198475838
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,2560,fp8,0.007304533695181212
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,2048,fp8,0.007168000315626462
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,1536,fp8,0.007065599535902341
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,1024,fp8,0.007196800286571186
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,768,fp8,0.007309866324067117
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,512,fp8,0.007133866722385089
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,256,fp8,0.007065599535902341
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,128,fp8,0.006997333591183027
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,64,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,32,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,80,65536,fp8,0.011844267447789509
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,16384,fp8,0.007953066378831863
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,12288,fp8,0.0077482670545578
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,10240,fp8,0.007680000116427739
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,8192,fp8,0.0074815995991230014
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,7168,fp8,0.007372800509134929
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,6144,fp8,0.007441066702206929
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,5120,fp8,0.007475200047095616
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,4096,fp8,0.007270400474468868
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,3584,fp8,0.007139200096329053
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,3072,fp8,0.0073727998882532125
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,2560,fp8,0.007168000315626462
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,2048,fp8,0.007133866970737775
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,1536,fp8,0.007168000315626462
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,1024,fp8,0.007137066746751468
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,768,fp8,0.007090133801102638
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,512,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,256,fp8,0.007139200096329053
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,128,fp8,0.007129600023229917
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,64,fp8,0.007075199857354163
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,32,fp8,0.007133866474032401
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,64,65536,fp8,0.01085439920425415
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,16384,fp8,0.007645866523186366
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,12288,fp8,0.0074410667022069285
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,10240,fp8,0.007475200295448304
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,8192,fp8,0.007436799754699071
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,7168,fp8,0.007474133496483166
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,6144,fp8,0.007441066453854243
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,5120,fp8,0.007270399605234465
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,4096,fp8,0.007406933481494586
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,3584,fp8,0.007202132915457091
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,3072,fp8,0.007231999809543292
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,2560,fp8,0.007236266508698463
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,2048,fp8,0.00724266655743122
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,1536,fp8,0.007236266508698463
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,1024,fp8,0.007194666440288226
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,768,fp8,0.007202132915457091
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,512,fp8,0.007065599535902341
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,256,fp8,0.007133866474032401
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,128,fp8,0.007073066507776577
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,64,fp8,0.007135999823609989
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,32,fp8,0.007133866722385089
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,48,65536,fp8,0.010008532802263895
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,16384,fp8,0.007432533552249272
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,12288,fp8,0.007611733426650364
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,10240,fp8,0.007577600081761678
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,8192,fp8,0.007275732979178429
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,7168,fp8,0.007168000315626462
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,6144,fp8,0.007270400101939837
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,5120,fp8,0.007338666295011839
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,4096,fp8,0.007193599517146747
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,3584,fp8,0.007231999809543292
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,3072,fp8,0.0072277331103881215
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,2560,fp8,0.007235200330615044
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,2048,fp8,0.007172267014781634
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,1536,fp8,0.0071018664787213
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,1024,fp8,0.007164799546202025
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,768,fp8,0.007133866722385089
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,512,fp8,0.007103999828298886
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,256,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,128,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,64,fp8,0.007168000315626462
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,32,fp8,0.007099732880791028
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,32,65536,fp8,0.009085866312185924
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,16384,fp8,0.007406933854023616
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,12288,fp8,0.007277866825461388
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,10240,fp8,0.0073376003652811045
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,8192,fp8,0.00713920034468174
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,7168,fp8,0.007226666187246641
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,6144,fp8,0.007203199590245884
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,5120,fp8,0.007168000067273776
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,4096,fp8,0.007133866722385089
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,3584,fp8,0.007129600023229917
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,3072,fp8,0.007162666693329811
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,2560,fp8,0.00723413365582625
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,2048,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,1536,fp8,0.007097600276271502
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,1024,fp8,0.007199999814232191
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,768,fp8,0.007133866722385089
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,512,fp8,0.007065599535902341
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,256,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,128,fp8,0.00713066669801871
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,64,fp8,0.007099732880791028
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,32,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,16,65536,fp8,0.007918933530648549
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,16384,fp8,0.007133866722385089
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,12288,fp8,0.007230932886401813
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,10240,fp8,0.007202132915457091
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,8192,fp8,0.007065599535902341
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,7168,fp8,0.007065599535902341
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,6144,fp8,0.007133866722385089
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,5120,fp8,0.007193600262204806
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,4096,fp8,0.007065599535902341
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,3584,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,3072,fp8,0.007202132915457091
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,2560,fp8,0.007065599535902341
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,2048,fp8,0.007099732880791028
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,1536,fp8,0.007026132817069691
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,1024,fp8,0.007031466191013654
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,768,fp8,0.007065599535902341
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,512,fp8,0.007065600280960401
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,256,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,128,fp8,0.007062399759888649
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,64,fp8,0.007099732880791028
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,32,fp8,0.007165866220990816
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,8,65536,fp8,0.007441066950559617
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,16384,fp8,0.007095466181635856
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,12288,fp8,0.0068607998390992474
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,10240,fp8,0.007202132915457091
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,8192,fp8,0.007133866474032401
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,7168,fp8,0.007133866474032401
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,6144,fp8,0.0070965331047773365
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,5120,fp8,0.007065599535902341
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,4096,fp8,0.007103999828298886
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,3584,fp8,0.007127466922005018
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,3072,fp8,0.00713279979924361
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,2560,fp8,0.007100799803932507
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,2048,fp8,0.007098666951060295
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,1536,fp8,0.007066666210691134
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,1024,fp8,0.007094399506847065
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,768,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,512,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,256,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,128,fp8,0.007133866474032401
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,64,fp8,0.007128532851735751
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,32,fp8,0.007099732880791028
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,4,65536,fp8,0.007304533074299495
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,16384,fp8,0.007162666444977124
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,12288,fp8,0.007100799803932507
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,10240,fp8,0.007133866722385089
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,8192,fp8,0.007126400247216224
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,7168,fp8,0.007102933153510093
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,6144,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,5120,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,4096,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,3584,fp8,0.007031466936071714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,3072,fp8,0.007062399759888648
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,2560,fp8,0.007065600280960401
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,2048,fp8,0.00706879956026872
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,1536,fp8,0.007099733377496402
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,1024,fp8,0.007159466668963431
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,768,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,512,fp8,0.007338666295011839
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,256,fp8,0.007133866474032401
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,128,fp8,0.007161599521835645
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,64,fp8,0.007133866474032401
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,32,fp8,0.007133866474032401
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,2,65536,fp8,0.007236266508698463
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,16384,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,12288,fp8,0.007133866722385089
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,10240,fp8,0.007135999823609989
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,8192,fp8,0.007099732880791028
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,7168,fp8,0.007099732880791028
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,6144,fp8,0.006826666990915935
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,5120,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,4096,fp8,0.007108266527454057
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,3584,fp8,0.007270399853587149
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,3072,fp8,0.007202133660515149
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,2560,fp8,0.007133866474032401
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,2048,fp8,0.007099733129143714
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,1536,fp8,0.007168000315626462
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,1024,fp8,0.007059199983874957
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,768,fp8,0.007065600280960401
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,512,fp8,0.007168000067273776
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,256,fp8,0.007202133660515149
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,128,fp8,0.007065600280960401
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,64,fp8,0.007168000067273776
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,32,fp8,0.007171199843287468
TRTLLM,1.0.0,NVIDIA L40S,compute_scale,torch_ops,1,65536,fp8,0.007125333324074745
