framework,version,device,op_name,kernel_source,m,k,quant_dtype,latency
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,16384,fp8,0.08959999084472658
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,12288,fp8,0.06750186284383136
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,10240,fp8,0.05665812492370606
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,8192,fp8,0.044869343439737974
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,7168,fp8,0.038188799222310374
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,6144,fp8,0.03060586452484132
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,5120,fp8,0.01971840063730876
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,4096,fp8,0.021198936303456622
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,3584,fp8,0.02773866653442382
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,3072,fp8,0.020870401461919146
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,2560,fp8,0.016318933169047034
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,2048,fp8,0.016126930713653564
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,1536,fp8,0.014283733566602074
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,1024,fp8,0.012132266163825988
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,768,fp8,0.012185599406560261
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,512,fp8,0.011477333803971607
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,256,fp8,0.011276799937089283
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,128,fp8,0.010467199981212618
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,64,fp8,0.008908799787362417
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,32,fp8,0.00910506621003151
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8192,65536,fp8,0.29637222290039067
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,16384,fp8,0.04765332539876305
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,12288,fp8,0.032573866844177235
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,10240,fp8,0.022710402806599944
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,8192,fp8,0.025347197055816652
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,7168,fp8,0.02926186720530191
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,6144,fp8,0.02265386978785197
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,5120,fp8,0.02034560044606526
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,4096,fp8,0.01749653220176697
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,3584,fp8,0.018674131234486902
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,3072,fp8,0.016615466276804606
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,2560,fp8,0.01602453589439392
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,2048,fp8,0.014425600568453472
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,1536,fp8,0.014876799782117206
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,1024,fp8,0.011813334623972575
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,768,fp8,0.013559466103712716
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,512,fp8,0.011911467214425405
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,256,fp8,0.011974400281906127
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,128,fp8,0.010482133676608402
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,64,fp8,0.00904319981733958
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,32,fp8,0.0100192000468572
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4096,65536,fp8,1.8068074544270833
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,16384,fp8,0.0779925346374512
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,12288,fp8,0.0676543951034546
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,10240,fp8,0.04218880335489908
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,8192,fp8,0.04145280122756958
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,7168,fp8,0.03221652905146281
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,6144,fp8,0.027134935061136886
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,5120,fp8,0.025431466102600095
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,4096,fp8,0.0227509339650472
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,3584,fp8,0.024143997828165696
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,3072,fp8,0.022078933318456017
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,2560,fp8,0.021218132972717286
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,2048,fp8,0.017310932278633114
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,1536,fp8,0.01818346579869588
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,1024,fp8,0.018855466445287072
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,768,fp8,0.014081066846847533
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,512,fp8,0.014730666577816008
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,256,fp8,0.0120128000775973
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,128,fp8,0.010319999357064564
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,64,fp8,0.01156800016760826
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,32,fp8,0.010966399808724723
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2048,65536,fp8,0.0
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,16384,fp8,0.041815471649169905
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,12288,fp8,0.03450133403142293
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,10240,fp8,0.027199999491373694
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,8192,fp8,0.024984534581502273
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,7168,fp8,0.02676373322804769
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,6144,fp8,0.027479465802510577
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,5120,fp8,0.019930668671925864
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,4096,fp8,0.017628799875577292
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,3584,fp8,0.017815465728441872
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,3072,fp8,0.02146773338317871
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,2560,fp8,0.015538133184115092
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,2048,fp8,0.015883733332157136
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,1536,fp8,0.014986666043599446
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,1024,fp8,0.01348693370819092
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,768,fp8,0.01321813364823659
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,512,fp8,0.012955733388662339
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,256,fp8,0.009261866410573326
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,128,fp8,0.009512532999118169
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,64,fp8,0.009959467003742853
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,32,fp8,0.009587200234333675
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1024,65536,fp8,0.05092159907023114
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,16384,fp8,0.014571732282638548
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,12288,fp8,0.014982399344444274
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,10240,fp8,0.013530667622884116
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,8192,fp8,0.014007468024889629
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,7168,fp8,0.21676905850569408
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,6144,fp8,0.018606932957967122
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,5120,fp8,0.016884267330169674
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,4096,fp8,0.01790826618671417
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,3584,fp8,0.01892800033092499
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,3072,fp8,0.015683200955390934
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,2560,fp8,0.012738132973512013
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,2048,fp8,0.01272639979918798
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,1536,fp8,0.012256000936031342
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,1024,fp8,0.012132267157236733
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,768,fp8,0.009453866879145303
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,512,fp8,0.009852799773216249
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,256,fp8,0.009561599791049957
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,128,fp8,0.008696533491214117
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,64,fp8,0.007796266674995423
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,32,fp8,0.0082218661904335
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,768,65536,fp8,0.030498139063517254
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,16384,fp8,0.012668801347414655
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,12288,fp8,0.012745599945386248
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,10240,fp8,0.011103999614715577
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,8192,fp8,0.010929066439469655
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,7168,fp8,0.011459200580914814
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,6144,fp8,0.011423999567826588
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,5120,fp8,0.01213546593983968
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,4096,fp8,0.011632000406583149
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,3584,fp8,0.010200533270835876
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,3072,fp8,0.009864533444245654
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,2560,fp8,0.010385067264238993
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,2048,fp8,0.010532266398270927
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,1536,fp8,0.010338133821884789
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,1024,fp8,0.009257599959770841
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,768,fp8,0.008850132922331493
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,512,fp8,0.009170133123795192
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,256,fp8,0.007451733201742172
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,128,fp8,0.007814400146404903
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,64,fp8,0.0102272001405557
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,32,fp8,0.00779093305269877
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,512,65536,fp8,0.019978666305542
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,16384,fp8,0.01276373267173767
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,12288,fp8,0.01025066723426183
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,10240,fp8,0.011050666868686675
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,8192,fp8,0.011512533823649088
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,7168,fp8,0.010575999816258747
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,6144,fp8,0.011036800344785056
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,5120,fp8,0.010231466591358186
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,4096,fp8,0.010954666634400687
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,3584,fp8,0.010981333255767823
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,3072,fp8,0.010211199522018433
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,2560,fp8,0.010614399860302607
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,2048,fp8,0.008796800176302595
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,1536,fp8,0.008667733023564021
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,1024,fp8,0.009505067020654678
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,768,fp8,0.009457066655158997
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,512,fp8,0.009584000458319981
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,256,fp8,0.007883733510971068
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,128,fp8,0.007743999610344569
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,64,fp8,0.008933333307504654
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,32,fp8,0.008217599987983704
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,384,65536,fp8,0.02097919980684916
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,16384,fp8,0.010971733927726745
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,12288,fp8,0.012113067011038462
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,10240,fp8,0.010552532474199931
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,8192,fp8,0.011458133657773336
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,7168,fp8,0.011455999811490375
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,6144,fp8,0.010022400319576264
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,5120,fp8,0.010401066641012827
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,4096,fp8,0.010899199793736138
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,3584,fp8,0.00965546667575836
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,3072,fp8,0.010075733065605164
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,2560,fp8,0.008867200215657553
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,2048,fp8,0.009026133765776953
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,1536,fp8,0.009329066673914591
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,1024,fp8,0.008680533121029534
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,768,fp8,0.009379200140635173
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,512,fp8,0.008043733487526575
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,256,fp8,0.008017066369454064
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,128,fp8,0.008316800246636074
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,64,fp8,0.007830399523178736
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,32,fp8,0.0077621338268121075
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,256,65536,fp8,0.01464959979057312
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,16384,fp8,0.012154667576154073
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,12288,fp8,0.011238400141398111
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,10240,fp8,0.011048534015814463
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,8192,fp8,0.009811200201511383
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,7168,fp8,0.0105322668949763
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,6144,fp8,0.010238933563232423
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,5120,fp8,0.009972266852855682
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,4096,fp8,0.010438400010267895
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,3584,fp8,0.009511466572682061
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,3072,fp8,0.009038933118184407
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,2560,fp8,0.009470933179060618
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,2048,fp8,0.009310933699210485
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,1536,fp8,0.007838933169841765
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,1024,fp8,0.009873066842556
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,768,fp8,0.010142933080593745
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,512,fp8,0.0080287995437781
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,256,fp8,0.008212266614039739
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,128,fp8,0.008524800340334576
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,64,fp8,0.007887999465068182
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,32,fp8,0.00819306621948878
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,192,65536,fp8,0.012745600938796998
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,16384,fp8,0.011144533753395082
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,12288,fp8,0.011493333677450819
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,10240,fp8,0.0101760004957517
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,8192,fp8,0.010043733318646747
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,7168,fp8,0.010609066983064016
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,6144,fp8,0.010504533598820368
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,5120,fp8,0.010132267077763876
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,4096,fp8,0.010414933413267134
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,3584,fp8,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,3072,fp8,0.008977066228787104
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,2560,fp8,0.009526399771372478
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,2048,fp8,0.009426133334636688
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,1536,fp8,0.009314132978518803
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,1024,fp8,0.00910293310880661
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,768,fp8,0.009405867258707682
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,512,fp8,0.007738666981458663
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,256,fp8,0.008113067100445428
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,128,fp8,0.00774933323264122
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,64,fp8,0.008232532938321432
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,32,fp8,0.008215466886758803
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,160,65536,fp8,0.01315520107746124
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,16384,fp8,0.011367466549078626
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,12288,fp8,0.010436266163984936
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,10240,fp8,0.010281599561373393
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,8192,fp8,0.011025066673755645
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,7168,fp8,0.010937600334485372
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,6144,fp8,0.008967466652393341
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,5120,fp8,0.009397332866986591
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,4096,fp8,0.009208533664544424
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,3584,fp8,0.009061333537101746
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,3072,fp8,0.01101333349943161
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,2560,fp8,0.00850239967306455
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,2048,fp8,0.01004266639550527
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,1536,fp8,0.009380266318718593
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,1024,fp8,0.00794986660281817
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,768,fp8,0.0079477330048879
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,512,fp8,0.007821866124868392
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,256,fp8,0.007799466451009115
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,128,fp8,0.008232533435026805
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,64,fp8,0.007868800063927968
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,32,fp8,0.008226133634646735
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,128,65536,fp8,0.012730666995048523
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,16384,fp8,0.009513599673906963
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,12288,fp8,0.010671999802192052
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,10240,fp8,0.01040106639266014
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,8192,fp8,0.00871360053618749
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,7168,fp8,0.009092266360918682
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,6144,fp8,0.00866453299919764
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,5120,fp8,0.009492266923189163
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,4096,fp8,0.009285333752632142
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,3584,fp8,0.00884159952402115
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,3072,fp8,0.009716266393661497
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,2560,fp8,0.009391999493042627
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,2048,fp8,0.009398400286833445
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,1536,fp8,0.007871999839941659
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,1024,fp8,0.009258667131265004
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,768,fp8,0.00888106698791186
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,512,fp8,0.007739733656247457
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,256,fp8,0.008249599983294805
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,128,fp8,0.008246399958928426
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,64,fp8,0.008202666292587917
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,32,fp8,0.008242133508125941
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,96,65536,fp8,0.012728532155354817
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,16384,fp8,0.010297599931557974
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,12288,fp8,0.010396800190210342
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,10240,fp8,0.009034666170676548
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,8192,fp8,0.00918293297290802
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,7168,fp8,0.009010133147239684
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,6144,fp8,0.009330133348703385
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,5120,fp8,0.00947519987821579
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,4096,fp8,0.008835199971993763
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,3584,fp8,0.00906026636560758
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,3072,fp8,0.008704000463088352
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,2560,fp8,0.008750933160384496
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,2048,fp8,0.00939733311533928
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,1536,fp8,0.007765333106120428
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,1024,fp8,0.00952426667014758
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,768,fp8,0.008036266267299651
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,512,fp8,0.007749333480993906
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,256,fp8,0.008264532933632534
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,128,fp8,0.0077919994791348784
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,64,fp8,0.007828266918659212
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,32,fp8,0.008165333420038223
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,80,65536,fp8,0.012074667215347291
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,16384,fp8,0.010813867052396137
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,12288,fp8,0.00859733298420906
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,10240,fp8,0.009433599561452864
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,8192,fp8,0.009463467200597129
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,7168,fp8,0.010492800176143647
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,6144,fp8,0.009248000135024387
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,5120,fp8,0.008384000261624655
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,4096,fp8,0.009117866804202398
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,3584,fp8,0.009251200159390767
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,3072,fp8,0.008916266759236654
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,2560,fp8,0.009345066795746485
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,2048,fp8,0.0088981327911218
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,1536,fp8,0.00974293351173401
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,1024,fp8,0.00828800027569135
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,768,fp8,0.007822933544715246
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,512,fp8,0.008331733196973801
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,256,fp8,0.007752533257007598
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,128,fp8,0.007804800073305766
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,64,fp8,0.008244267106056212
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,32,fp8,0.00782186711827914
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,64,65536,fp8,0.011349333326021832
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,16384,fp8,0.009305600573619205
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,12288,fp8,0.008870400240023932
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,10240,fp8,0.009172266473372779
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,8192,fp8,0.00901440034310023
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,7168,fp8,0.008821333448092144
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,6144,fp8,0.009058133016029995
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,5120,fp8,0.008714666962623597
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,4096,fp8,0.009723733613888423
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,3584,fp8,0.009841067095597587
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,3072,fp8,0.008634666353464127
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,2560,fp8,0.008660266548395159
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,2048,fp8,0.007875200112660727
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,1536,fp8,0.009113600105047226
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,1024,fp8,0.007838933169841767
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,768,fp8,0.008248533308506011
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,512,fp8,0.007758933802445729
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,256,fp8,0.007752533753712973
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,128,fp8,0.00821333353718122
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,64,fp8,0.007726932813723881
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,32,fp8,0.008225066214799879
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,48,65536,fp8,0.012152533233165742
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,16384,fp8,0.00893653333187103
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,12288,fp8,0.009293866654237112
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,10240,fp8,0.008931199709574383
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,8192,fp8,0.008195200065771738
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,7168,fp8,0.008513066669305165
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,6144,fp8,0.009393066912889481
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,5120,fp8,0.007701333363850912
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,4096,fp8,0.0074581332504749295
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,3584,fp8,0.008222933610280356
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,3072,fp8,0.009181866546471913
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,2560,fp8,0.009671466797590256
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,2048,fp8,0.008177066346009574
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,1536,fp8,0.008778666704893114
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,1024,fp8,0.008261332909266153
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,768,fp8,0.008811732878287633
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,512,fp8,0.0078079996009667715
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,256,fp8,0.008235733211040498
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,128,fp8,0.007822933048009873
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,64,fp8,0.00816639984647433
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,32,fp8,0.008154666672150295
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,32,65536,fp8,0.01102826644976934
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,16384,fp8,0.008902399490276974
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,12288,fp8,0.009395199765761694
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,10240,fp8,0.009163733323415121
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,8192,fp8,0.007831466694672903
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,7168,fp8,0.008060800035794576
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,6144,fp8,0.0077781329552332554
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,5120,fp8,0.008962133278449377
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,4096,fp8,0.008549333612124127
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,3584,fp8,0.008547200262546541
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,3072,fp8,0.008380799988905589
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,2560,fp8,0.008225066959857941
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,2048,fp8,0.007896533608436583
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,1536,fp8,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,1024,fp8,0.007810132950544356
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,768,fp8,0.007807999849319457
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,512,fp8,0.008231466511885326
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,256,fp8,0.007837866743405659
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,128,fp8,0.008154666672150295
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,64,fp8,0.008181332796812059
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,32,fp8,0.007747199883063634
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,16,65536,fp8,0.011914666493733725
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,16384,fp8,0.007461333523193996
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,12288,fp8,0.008083199958006541
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,10240,fp8,0.007878400137027104
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,8192,fp8,0.007826133320728939
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,7168,fp8,0.009900800138711929
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,6144,fp8,0.007982933024565379
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,5120,fp8,0.007725866883993147
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,4096,fp8,0.008539733539024989
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,3584,fp8,0.007889067133267721
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,3072,fp8,0.007795199503501257
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,2560,fp8,0.008226133634646735
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,2048,fp8,0.008231466263532638
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,1536,fp8,0.008092799534400305
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,1024,fp8,0.007768533875544865
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,768,fp8,0.008196266243855157
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,512,fp8,0.007749333232641221
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,256,fp8,0.008219732840855917
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,128,fp8,0.008142933249473572
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,64,fp8,0.007764267176389693
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,32,fp8,0.008209066092967985
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,8,65536,fp8,0.008850133419036867
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,16384,fp8,0.007809066275755566
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,12288,fp8,0.00914773295323054
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,10240,fp8,0.007814400146404901
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,8192,fp8,0.008232533186674119
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,7168,fp8,0.00876800020535787
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,6144,fp8,0.007815466324488322
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,5120,fp8,0.00835840031504631
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,4096,fp8,0.0078165332476298
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,3584,fp8,0.008174932996431985
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,3072,fp8,0.008699733018875121
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,2560,fp8,0.0077589333057403575
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,2048,fp8,0.008250666409730913
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,1536,fp8,0.008218666662772496
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,1024,fp8,0.007764266928037007
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,768,fp8,0.008225066711505254
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,512,fp8,0.007792000224192937
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,256,fp8,0.00775893380244573
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,128,fp8,0.00815573384364446
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,64,fp8,0.007847466071446737
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,32,fp8,0.008205867062012356
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,4,65536,fp8,0.009350400169690451
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,16384,fp8,0.007754667103290558
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,12288,fp8,0.007873066514730454
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,10240,fp8,0.008228266239166261
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,8192,fp8,0.008262399584054947
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,7168,fp8,0.007772800077994665
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,6144,fp8,0.008415999760230383
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,5120,fp8,0.007772800326347352
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,4096,fp8,0.007766400029261907
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,3584,fp8,0.008231467008590699
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,3072,fp8,0.0077877330283323935
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,2560,fp8,0.008247466882069905
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,2048,fp8,0.008188800265391668
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,1536,fp8,0.007739733159542083
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,1024,fp8,0.008233600109815598
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,768,fp8,0.007765332857767741
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,512,fp8,0.007708799839019776
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,256,fp8,0.008178133269151053
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,128,fp8,0.0077514665822188055
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,64,fp8,0.00821546639005343
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,32,fp8,0.008203733215729396
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,2,65536,fp8,0.008425599833329519
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,16384,fp8,0.008221866935491562
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,12288,fp8,0.007742933680613836
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,10240,fp8,0.007740800331036249
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,8192,fp8,0.00824746663371722
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,7168,fp8,0.007805866748094558
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,6144,fp8,0.008230399837096532
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,5120,fp8,0.008214400211970011
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,4096,fp8,0.0077471996347109485
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,3584,fp8,0.008232532938321432
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,3072,fp8,0.007723733037710191
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,2560,fp8,0.007750399659077327
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,2048,fp8,0.00820479989051819
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,1536,fp8,0.008194132894277572
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,1024,fp8,0.00774826630949974
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,768,fp8,0.007747199883063634
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,512,fp8,0.00819733291864395
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,256,fp8,0.007727999488512674
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,128,fp8,0.008233600358168284
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,64,fp8,0.00821866641441981
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,32,fp8,0.007724799712498983
TRTLLM,1.2.0rc5,NVIDIA B200,compute_scale,torch_ops,1,65536,fp8,0.00857493331034978
