framework,version,device,op_name,kernel_source,m,k,quant_dtype,latency
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,16384,fp8,0.27679678599039714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,12288,fp8,0.20837440490722656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,10240,fp8,0.17464106877644855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,8192,fp8,0.14057386716206868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,7168,fp8,0.12333760261535645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,6144,fp8,0.1060970703760783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,5120,fp8,0.08924586772918701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,4096,fp8,0.07204480171203613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,3584,fp8,0.06356906493504842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,3072,fp8,0.05505706469217936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,2560,fp8,0.04644266764322917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,2048,fp8,0.037723731994628903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,1536,fp8,0.020351999998092653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,1024,fp8,0.012735999623934426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,768,fp8,0.009991466999053955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,512,fp8,0.007381333410739899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,256,fp8,0.004680533210436503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,128,fp8,0.0037717332442601522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,64,fp8,0.003236266722281774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,32,fp8,0.0034730667869249977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,65536,fp8,1.1105514526367188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,16384,fp8,0.14051307042439778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,12288,fp8,0.10623040199279785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,10240,fp8,0.08940906524658203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,8192,fp8,0.07220266660054525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,7168,fp8,0.06369706789652506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,6144,fp8,0.05505706469217936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,5120,fp8,0.04640320142110189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,4096,fp8,0.037519999345143634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,3584,fp8,0.030577067534128827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,3072,fp8,0.020925867557525634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,2560,fp8,0.015449600418408713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,2048,fp8,0.012682666381200155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,1536,fp8,0.009969066580136616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,1024,fp8,0.00740479975938797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,768,fp8,0.005972266693909963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,512,fp8,0.0047541335225105286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,256,fp8,0.003673599908749262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,128,fp8,0.0032768001159032187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,64,fp8,0.002977066735426585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,32,fp8,0.002765866617361705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,65536,fp8,0.5534112294514973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,16384,fp8,0.07210133075714112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,12288,fp8,0.05510933399200439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,10240,fp8,0.04638186693191528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,8192,fp8,0.03794453144073486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,7168,fp8,0.03113173246383667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,6144,fp8,0.020066134134928384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,5120,fp8,0.015410133202870688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,4096,fp8,0.0127210666735967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,3584,fp8,0.011292800307273865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,3072,fp8,0.010059733192125957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,2560,fp8,0.008634666601816814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,2048,fp8,0.007358933488527934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,1536,fp8,0.006009600063165029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,1024,fp8,0.004760533571243286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,768,fp8,0.004189866781234741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,512,fp8,0.003651199986537298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,256,fp8,0.0033088001112143196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,128,fp8,0.002979200085004171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,64,fp8,0.0028384000062942503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,32,fp8,0.002762666592995326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,65536,fp8,0.27729279200236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,16384,fp8,0.03779413302739461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,12288,fp8,0.020539732774098714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,10240,fp8,0.01535146633783976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,8192,fp8,0.012618666887283326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,7168,fp8,0.011370666821797689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,6144,fp8,0.010006399949391682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,5120,fp8,0.008711466193199157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,4096,fp8,0.007316266496976216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,3584,fp8,0.0067093332608540845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,3072,fp8,0.0060362666845321655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,2560,fp8,0.005298133194446564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,2048,fp8,0.004760533571243286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,1536,fp8,0.004153600086768469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,1024,fp8,0.003755733370780945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,768,fp8,0.0034272000193595886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,512,fp8,0.0032992000381151833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,256,fp8,0.0029898665845394133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,128,fp8,0.0028512001037597655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,64,fp8,0.0027744000156720476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,32,fp8,0.002794666588306427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,65536,fp8,0.140229336420695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,16384,fp8,0.020538665850957236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,12288,fp8,0.013929599523544311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,10240,fp8,0.012060800194740295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,8192,fp8,0.009964799880981446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,7168,fp8,0.008975999553998311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,6144,fp8,0.008065066734949748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,5120,fp8,0.006964266796906789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,4096,fp8,0.006045866509278615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,3584,fp8,0.005418666700522105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,3072,fp8,0.005065600077311197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,2560,fp8,0.004561066627502441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,2048,fp8,0.004181333382924398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,1536,fp8,0.0038762666285037995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,1024,fp8,0.003538133452335993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,768,fp8,0.0033098667860031127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,512,fp8,0.003205333401759466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,256,fp8,0.002796799937884013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,128,fp8,0.0028522667785485585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,64,fp8,0.0027829334139823914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,32,fp8,0.0028373333315054577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,65536,fp8,0.10615146954854329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,16384,fp8,0.012662399808565775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,12288,fp8,0.010053333640098572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,10240,fp8,0.00865280032157898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,8192,fp8,0.007421866556008657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,7168,fp8,0.006640000144640605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,6144,fp8,0.006117333471775055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,5120,fp8,0.005341866612434387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,4096,fp8,0.004738133152325948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,3584,fp8,0.004399999976158142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,3072,fp8,0.004229333500067393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,2560,fp8,0.0039381332695484165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,2048,fp8,0.0037759999434153237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,1536,fp8,0.003475199888149897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,1024,fp8,0.0033290666838486993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,768,fp8,0.0031136001149813337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,512,fp8,0.003009066730737686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,256,fp8,0.002890666574239731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,128,fp8,0.002755200117826462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,64,fp8,0.0028704000016053517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,32,fp8,0.00276799996693929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,65536,fp8,0.07235199610392253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,16384,fp8,0.0100000003973643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,12288,fp8,0.008061866462230682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,10240,fp8,0.00710399995247523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,8192,fp8,0.006004266440868378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,7168,fp8,0.0055189331372578945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,6144,fp8,0.005005866785844167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,5120,fp8,0.004653866589069367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,4096,fp8,0.004147200038035711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,3584,fp8,0.004103466620047887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,3072,fp8,0.0038560000558694207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,2560,fp8,0.003676799933115641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,2048,fp8,0.0035466666022936503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,1536,fp8,0.0033088001112143196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,1024,fp8,0.003223466624816259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,768,fp8,0.003047466774781545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,512,fp8,0.002898133297761281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,256,fp8,0.002784000088771184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,128,fp8,0.002833066632350286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,64,fp8,0.0027488000690937043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,32,fp8,0.0028277332584063213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,65536,fp8,0.05493226846059164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,16384,fp8,0.007413333157698314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,12288,fp8,0.005986133217811584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,10240,fp8,0.005398400127887726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,8192,fp8,0.004704000055789947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,7168,fp8,0.0044714664419492085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,6144,fp8,0.0042026668787002565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,5120,fp8,0.003917866696914038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,4096,fp8,0.0037685332198937735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,3584,fp8,0.0036447999378045404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,3072,fp8,0.003533866753180822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,2560,fp8,0.003403733422358831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,2048,fp8,0.0033002667129039764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,1536,fp8,0.00314026673634847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,1024,fp8,0.003068800022204717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,768,fp8,0.002811733384927114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,512,fp8,0.002834133307139079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,256,fp8,0.0027402666707833606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,128,fp8,0.0028160000840822858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,64,fp8,0.002733866622050603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,32,fp8,0.0028149334092934927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,65536,fp8,0.03784960110982259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,16384,fp8,0.005990399916966756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,12288,fp8,0.005081599950790406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,10240,fp8,0.004593066871166229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,8192,fp8,0.00421013335386912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,7168,fp8,0.003996799886226654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,6144,fp8,0.003958400090535482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,5120,fp8,0.0036703998843828833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,4096,fp8,0.0035157332817713416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,3584,fp8,0.0033952000240484873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,3072,fp8,0.0033621333539485933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,2560,fp8,0.0032032000521818793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,2048,fp8,0.003219199925661087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,1536,fp8,0.002994133283694585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,1024,fp8,0.0028778667251269023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,768,fp8,0.002762666592995326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,512,fp8,0.002829866607983907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,256,fp8,0.0028064000109831494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,128,fp8,0.0027488000690937043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,64,fp8,0.0028223998844623564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,32,fp8,0.0027189334233601887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,65536,fp8,0.02035520076751709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,16384,fp8,0.005329066514968872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,12288,fp8,0.004660266637802124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,10240,fp8,0.004252799848715464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,8192,fp8,0.0039264000952243805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,7168,fp8,0.00391146664818128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,6144,fp8,0.003673599908749262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,5120,fp8,0.003572266548871994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,4096,fp8,0.0033728001018365227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,3584,fp8,0.003352533280849457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,3072,fp8,0.003189333279927572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,2560,fp8,0.0032287999987602235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,2048,fp8,0.0031167998909950255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,1536,fp8,0.002921599894762039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,1024,fp8,0.0028533334533373516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,768,fp8,0.0027637332677841187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,512,fp8,0.0028064000109831494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,256,fp8,0.0027232001225153605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,128,fp8,0.0028053333361943563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,64,fp8,0.0027295999228954316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,32,fp8,0.0028234665592511495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,65536,fp8,0.015336533387502035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,16384,fp8,0.004789333542188009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,12288,fp8,0.0041461333632469176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,10240,fp8,0.003977599988381068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,8192,fp8,0.003673599908749262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,7168,fp8,0.003692800054947535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,6144,fp8,0.0034613333642482757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,5120,fp8,0.0033845332761605583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,4096,fp8,0.0032960000137488045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,3584,fp8,0.0031626666585604347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,3072,fp8,0.0032042667269706728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,2560,fp8,0.0030517332255840302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,2048,fp8,0.003067733347415924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,1536,fp8,0.0028192001084486645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,1024,fp8,0.0028437333802382152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,768,fp8,0.0028010666370391845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,512,fp8,0.002846933404604594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,256,fp8,0.0027477333943049112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,128,fp8,0.002812800059715907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,64,fp8,0.0027594665686289472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,32,fp8,0.0028319999575614927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,65536,fp8,0.012737066547075907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,16384,fp8,0.004248533149560293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,12288,fp8,0.003972266614437103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,10240,fp8,0.003681066632270813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,8192,fp8,0.0035455999275048576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,7168,fp8,0.003425066669782003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,6144,fp8,0.0033962666988372804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,5120,fp8,0.00322026660044988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,4096,fp8,0.0032245332996050516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,3584,fp8,0.0030847998956839246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,3072,fp8,0.0031082667410373688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,2560,fp8,0.002924799919128418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,2048,fp8,0.0030154667794704436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,1536,fp8,0.002812800059715907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,1024,fp8,0.0028575999041398365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,768,fp8,0.0027797333896160126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,512,fp8,0.002825599908828735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,256,fp8,0.0027306665976842242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,128,fp8,0.0027456000447273255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,64,fp8,0.002865066627661387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,32,fp8,0.00275093341867129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,65536,fp8,0.010406399766604107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,16384,fp8,0.0039434666434923805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,12288,fp8,0.003802666564782461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,10240,fp8,0.003517866631348928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,8192,fp8,0.0034773332377274835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,7168,fp8,0.003291733314593633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,6144,fp8,0.0033151999115943907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,5120,fp8,0.0031669333577156065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,4096,fp8,0.0031583999594052637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,3584,fp8,0.0030378667016824085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,3072,fp8,0.0030229332546393077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,2560,fp8,0.0028522667785485585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,2048,fp8,0.0028917332490285236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,1536,fp8,0.0028607999285062153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,1024,fp8,0.002809600035349528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,768,fp8,0.002869333326816559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,512,fp8,0.0027562665442625684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,256,fp8,0.0029077333708604175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,128,fp8,0.0027562665442625684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,64,fp8,0.0028170667588710784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,32,fp8,0.0027594665686289472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,65536,fp8,0.008779733379681905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,16384,fp8,0.003681066632270813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,12288,fp8,0.003555200000603994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,10240,fp8,0.003401600072781245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,8192,fp8,0.0033386667569478357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,7168,fp8,0.0031925333042939507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,6144,fp8,0.0032159999012947083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,5120,fp8,0.0030933332939942675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,4096,fp8,0.0030229332546393077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,3584,fp8,0.0029685333371162414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,3072,fp8,0.002855466554562251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,2560,fp8,0.00288426677385966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,2048,fp8,0.002799999962250392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,1536,fp8,0.002844800055027008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,1024,fp8,0.00276053324341774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,768,fp8,0.0028042666614055633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,512,fp8,0.0027285332481066385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,256,fp8,0.0028042666614055633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,128,fp8,0.0027295999228954316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,64,fp8,0.002812800059715907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,32,fp8,0.0027104000250498454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,65536,fp8,0.007458133498827617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,16384,fp8,0.0034688000877698263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,12288,fp8,0.0033749334514141084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,10240,fp8,0.0032810665667057036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,8192,fp8,0.003126399964094162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,7168,fp8,0.0031413334111372627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,6144,fp8,0.0030016000072161358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,5120,fp8,0.002998399982849757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,4096,fp8,0.002792533238728841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,3584,fp8,0.0028736000259717304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,3072,fp8,0.00276799996693929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,2560,fp8,0.002868266652027766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,2048,fp8,0.0027413333455721537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,1536,fp8,0.0028234665592511495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,1024,fp8,0.002726399898529053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,768,fp8,0.0027850667635599775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,512,fp8,0.0027285332481066385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,256,fp8,0.0028021333118279776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,128,fp8,0.0027221334477265675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,64,fp8,0.0027242665489514667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,32,fp8,0.0028021333118279776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,65536,fp8,0.006037333110968272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,16384,fp8,0.003307733436425527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,12288,fp8,0.0031114667654037476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,10240,fp8,0.003142400085926056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,8192,fp8,0.0029696000119050344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,7168,fp8,0.0029397333661715193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,6144,fp8,0.0028949332733949023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,5120,fp8,0.0027893332143624624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,4096,fp8,0.0028277332584063213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,3584,fp8,0.002808533360560735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,3072,fp8,0.0028512001037597655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,2560,fp8,0.0027424000203609467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,2048,fp8,0.0028181334336598715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,1536,fp8,0.0027295999228954316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,1024,fp8,0.002721066772937775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,768,fp8,0.0028010666370391845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,512,fp8,0.0027232001225153605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,256,fp8,0.0028031999866167706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,128,fp8,0.0027306665976842242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,64,fp8,0.00278613343834877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,32,fp8,0.0027274665733178455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,65536,fp8,0.004758400221665701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,16384,fp8,0.0029781334102153777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,12288,fp8,0.002890666574239731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,10240,fp8,0.00278613343834877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,8192,fp8,0.002856533229351044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,7168,fp8,0.002754133443037669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,6144,fp8,0.0028490667541821797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,5120,fp8,0.0027530667682488757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,4096,fp8,0.002834133307139079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,3584,fp8,0.0028223998844623564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,3072,fp8,0.0027413333455721537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,2560,fp8,0.002807466685771942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,2048,fp8,0.0027232001225153605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,1536,fp8,0.002808533360560735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,1024,fp8,0.002734933296839396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,768,fp8,0.002792533238728841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,512,fp8,0.002775466690460841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,256,fp8,0.0028223998844623564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,128,fp8,0.002776533365249634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,64,fp8,0.0028266665836175283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,32,fp8,0.002776533365249634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,65536,fp8,0.003773866593837738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,16384,fp8,0.002784000088771184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,12288,fp8,0.0028490667541821797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,10240,fp8,0.002775466690460841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,8192,fp8,0.0027776000400384264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,7168,fp8,0.0028437333802382152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,6144,fp8,0.00275093341867129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,5120,fp8,0.002850133428970973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,4096,fp8,0.002780800064404805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,3584,fp8,0.002811733384927114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,3072,fp8,0.002752000093460083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,2560,fp8,0.0028181334336598715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,2048,fp8,0.002757333219051361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,1536,fp8,0.0028480000793933867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,1024,fp8,0.002754133443037669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,768,fp8,0.0028234665592511495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,512,fp8,0.0027413333455721537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,256,fp8,0.002828799933195114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,128,fp8,0.0027615999182065325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,64,fp8,0.0028277332584063213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,32,fp8,0.002834133307139079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,65536,fp8,0.0032490665713946023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,16384,fp8,0.0028384000062942503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,12288,fp8,0.002757333219051361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,10240,fp8,0.0028373333315054577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,8192,fp8,0.002746666719516118
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,7168,fp8,0.0028213332096735638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,6144,fp8,0.002771199991305669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,5120,fp8,0.002757333219051361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,4096,fp8,0.0028277332584063213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,3584,fp8,0.0027445333699385325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,3072,fp8,0.0028266665836175283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,2560,fp8,0.002776533365249634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,2048,fp8,0.0028234665592511495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,1536,fp8,0.0027562665442625684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,1024,fp8,0.0028405333558718365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,768,fp8,0.0030293333033720653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,512,fp8,0.0027850667635599775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,256,fp8,0.0028192001084486645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,128,fp8,0.0027647999425729113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,64,fp8,0.002828799933195114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,32,fp8,0.0027477333943049112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,65536,fp8,0.003081600119670232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,16384,fp8,0.002775466690460841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,12288,fp8,0.0028234665592511495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,10240,fp8,0.002799999962250392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,8192,fp8,0.002829866607983907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,7168,fp8,0.0027562665442625684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,6144,fp8,0.002811733384927114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,5120,fp8,0.002765866617361705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,4096,fp8,0.0028362666567166646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,3584,fp8,0.002791466563940048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,3072,fp8,0.002765866617361705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,2560,fp8,0.0028618666032950084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,2048,fp8,0.0027562665442625684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,1536,fp8,0.0028277332584063213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,1024,fp8,0.002749866743882497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,768,fp8,0.0028384000062942503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,512,fp8,0.0027615999182065325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,256,fp8,0.0028319999575614927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,128,fp8,0.0027274665733178455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,64,fp8,0.002808533360560735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,32,fp8,0.002716800073782603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,65536,fp8,0.0028480000793933867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,16384,fp8,0.0027242665489514667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,12288,fp8,0.002807466685771942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,10240,fp8,0.0027242665489514667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,8192,fp8,0.002829866607983907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,7168,fp8,0.0027882667879263563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,6144,fp8,0.002980266759792964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,5120,fp8,0.0028234665592511495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,4096,fp8,0.002734933296839396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,3584,fp8,0.00277866671482722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,3072,fp8,0.0027189334233601887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,2560,fp8,0.00279573326309522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,2048,fp8,0.002733866622050603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,1536,fp8,0.0028149334092934927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,1024,fp8,0.0027285332481066385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,768,fp8,0.002791466563940048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,512,fp8,0.0027232001225153605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,256,fp8,0.0028149334092934927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,128,fp8,0.002755200117826462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,64,fp8,0.0028064000109831494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,32,fp8,0.0027253332237402597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,65536,fp8,0.002733866622050603
