framework,version,device,op_name,kernel_source,m,k,quant_dtype,latency
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,16384,fp8,0.20262932777404785
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,12288,fp8,0.1545450528462728
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,10240,fp8,0.1306122620900472
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,8192,fp8,0.10547946294148763
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,7168,fp8,0.09279680252075197
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,6144,fp8,0.08020692666371665
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,5120,fp8,0.06752533117930096
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,4096,fp8,0.05386880238850911
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,3584,fp8,0.046694401899973544
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,3072,fp8,0.03952639897664389
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,2560,fp8,0.03051413297653198
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,2048,fp8,0.023277866840362545
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,1536,fp8,0.020422399044036865
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,1024,fp8,0.013212801019350686
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,768,fp8,0.011243732770284018
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,512,fp8,0.009596800307432811
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,256,fp8,0.00849173367023468
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,128,fp8,0.007529599964618683
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,64,fp8,0.007257600128650666
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,32,fp8,0.007057066261768341
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8192,65536,fp8,0.7864566167195639
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,16384,fp8,0.10550612608591717
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,12288,fp8,0.08062294324239096
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,10240,fp8,0.06721919377644855
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,8192,fp8,0.054139733314514146
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,7168,fp8,0.04695146481196086
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,6144,fp8,0.03991680145263672
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,5120,fp8,0.029910401503245046
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,4096,fp8,0.02346880038579305
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,3584,fp8,0.024391466379165647
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,3072,fp8,0.020245331525802615
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,2560,fp8,0.015185067057609561
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,2048,fp8,0.012413867314656575
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,1536,fp8,0.011244799693425494
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,1024,fp8,0.009511467814445496
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,768,fp8,0.00915306657552719
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,512,fp8,0.0083338662981987
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,256,fp8,0.007579733679691952
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,128,fp8,0.007568000008662541
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,64,fp8,0.006842667112747827
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,32,fp8,0.006731733431418736
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4096,65536,fp8,0.3980309168497722
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,16384,fp8,0.053930672009785977
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,12288,fp8,0.03939519723256429
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,10240,fp8,0.030200537045796713
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,8192,fp8,0.022476800282796222
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,7168,fp8,0.025040000677108765
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,6144,fp8,0.020599464575449627
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,5120,fp8,0.015569067001342776
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,4096,fp8,0.012261333068211873
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,3584,fp8,0.01204906702041626
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,3072,fp8,0.010650667548179628
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,2560,fp8,0.0102101335922877
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,2048,fp8,0.009752534329891205
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,1536,fp8,0.00890240023533503
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,1024,fp8,0.008397866288820903
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,768,fp8,0.007990400244792303
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,512,fp8,0.007636267195145288
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,256,fp8,0.007320533196131389
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,128,fp8,0.006878933558861414
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,64,fp8,0.006771199901898703
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,32,fp8,0.006941867123047511
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2048,65536,fp8,0.2034858544667562
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,16384,fp8,0.02437546650568645
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,12288,fp8,0.020347734292348228
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,10240,fp8,0.015523200233777365
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,8192,fp8,0.012774398922920222
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,7168,fp8,0.011881600817044575
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,6144,fp8,0.01103679935137431
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,5120,fp8,0.010698666175206501
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,4096,fp8,0.009562666714191434
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,3584,fp8,0.009575466314951576
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,3072,fp8,0.00914773295323054
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,2560,fp8,0.008513066669305167
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,2048,fp8,0.008440533280372618
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,1536,fp8,0.008755199611186981
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,1024,fp8,0.0075040005147457125
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,768,fp8,0.007735466708739598
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,512,fp8,0.007412266731262208
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,256,fp8,0.006936533500750859
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,128,fp8,0.006767999877532323
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,64,fp8,0.006918399780988693
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,32,fp8,0.006822400043408077
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1024,65536,fp8,0.10573013623555502
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,16384,fp8,0.020838399728139237
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,12288,fp8,0.0141728013753891
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,10240,fp8,0.012051199873288475
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,8192,fp8,0.010817066828409829
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,7168,fp8,0.010490665833155315
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,6144,fp8,0.010208000739415487
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,5120,fp8,0.009427199761072796
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,4096,fp8,0.009009066720803578
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,3584,fp8,0.008665599425633747
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,3072,fp8,0.008477866152922313
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,2560,fp8,0.008198399841785432
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,2048,fp8,0.008130133152008057
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,1536,fp8,0.008010666817426682
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,1024,fp8,0.007606399804353714
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,768,fp8,0.007637332876523336
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,512,fp8,0.007103999704122543
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,256,fp8,0.007315199822187423
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,128,fp8,0.007155199845631918
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,64,fp8,0.006830932945013046
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,32,fp8,0.006902399907509486
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,768,65536,fp8,0.0810197353363037
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,16384,fp8,0.012338132659594216
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,12288,fp8,0.011400533715883891
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,10240,fp8,0.01053760051727295
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,8192,fp8,0.009685334066549938
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,7168,fp8,0.009526400268077852
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,6144,fp8,0.008835199971993765
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,5120,fp8,0.008849066495895386
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,4096,fp8,0.008931199709574383
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,3584,fp8,0.00853546659151713
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,3072,fp8,0.007741867005825044
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,2560,fp8,0.007990399996439617
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,2048,fp8,0.007879466315110523
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,1536,fp8,0.00770666648944219
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,1024,fp8,0.007526399940252304
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,768,fp8,0.007467733323574065
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,512,fp8,0.007026133437951406
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,256,fp8,0.007143466671307882
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,128,fp8,0.006903466333945591
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,64,fp8,0.0069909334182739254
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,32,fp8,0.00687360018491745
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,512,65536,fp8,0.0544106642405192
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,16384,fp8,0.011310934027036031
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,12288,fp8,0.010019199550151825
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,10240,fp8,0.009352533519268038
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,8192,fp8,0.009020799895127614
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,7168,fp8,0.009269333879152932
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,6144,fp8,0.00906773308912913
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,5120,fp8,0.008215466638406117
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,4096,fp8,0.0077120006084442135
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,3584,fp8,0.008254933108886083
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,3072,fp8,0.007650133470694225
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,2560,fp8,0.007337600489457448
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,2048,fp8,0.007474133123954138
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,1536,fp8,0.007499733318885167
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,1024,fp8,0.007593600451946259
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,768,fp8,0.007137066374222439
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,512,fp8,0.006954666475454966
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,256,fp8,0.007116799801588058
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,128,fp8,0.006731732934713363
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,64,fp8,0.0069322668015956886
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,32,fp8,0.006599466254313787
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,384,65536,fp8,0.0391765276590983
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,16384,fp8,0.00942720075448354
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,12288,fp8,0.008804266651471458
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,10240,fp8,0.008944000303745269
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,8192,fp8,0.008173867066701253
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,7168,fp8,0.008224000036716462
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,6144,fp8,0.008138666301965712
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,5120,fp8,0.007428266604741415
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,4096,fp8,0.0076736000676949825
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,3584,fp8,0.007550933212041855
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,3072,fp8,0.007150933643182118
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,2560,fp8,0.007433599481980005
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,2048,fp8,0.007565866659084956
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,1536,fp8,0.00731626624862353
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,1024,fp8,0.007201066613197327
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,768,fp8,0.007074133058389028
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,512,fp8,0.007038933783769606
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,256,fp8,0.007092266778151194
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,128,fp8,0.006856533139944076
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,64,fp8,0.006649600466092428
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,32,fp8,0.006933333724737168
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,256,65536,fp8,0.023660800854365026
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,16384,fp8,0.00918186604976654
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,12288,fp8,0.008338133494059244
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,10240,fp8,0.008136533200740816
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,8192,fp8,0.009057066092888515
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,7168,fp8,0.007922133058309556
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,6144,fp8,0.0076533332467079145
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,5120,fp8,0.007899733384450276
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,4096,fp8,0.0074879998962084455
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,3584,fp8,0.0076053328812122345
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,3072,fp8,0.007636266201734542
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,2560,fp8,0.007265066852172215
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,2048,fp8,0.0070773335794607805
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,1536,fp8,0.007133866349856058
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,1024,fp8,0.007384532938400905
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,768,fp8,0.0067605334023634585
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,512,fp8,0.007411200056473414
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,256,fp8,0.006964266796906789
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,128,fp8,0.006726399809122085
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,64,fp8,0.0069343999028205865
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,32,fp8,0.006690133114655813
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,192,65536,fp8,0.020533333222071334
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,16384,fp8,0.008877866963545481
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,12288,fp8,0.008266666531562807
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,10240,fp8,0.008085333307584127
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,8192,fp8,0.008026666442553202
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,7168,fp8,0.008196266492207845
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,6144,fp8,0.00757546698053678
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,5120,fp8,0.007363199939330419
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,4096,fp8,0.007116800298293433
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,3584,fp8,0.007887999961773555
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,3072,fp8,0.007356800138950346
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,2560,fp8,0.007388800382614137
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,2048,fp8,0.0072736000021298725
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,1536,fp8,0.0071104002495606744
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,1024,fp8,0.00730666642387708
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,768,fp8,0.006898133208354315
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,512,fp8,0.007169066617886224
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,256,fp8,0.007205333312352499
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,128,fp8,0.00666986679037412
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,64,fp8,0.006920533130566279
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,32,fp8,0.006693332890669504
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,160,65536,fp8,0.014742400248845416
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,16384,fp8,0.008130133152008057
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,12288,fp8,0.008125866452852884
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,10240,fp8,0.007959466427564621
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,8192,fp8,0.007678933193286261
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,7168,fp8,0.00795626665155093
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,6144,fp8,0.0074261335035165155
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,5120,fp8,0.007239466160535812
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,4096,fp8,0.007842133690913519
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,3584,fp8,0.007634133845567702
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,3072,fp8,0.007541332890590032
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,2560,fp8,0.007310933371384938
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,2048,fp8,0.0068746668597062435
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,1536,fp8,0.007235199958086014
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,1024,fp8,0.0071690671145915996
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,768,fp8,0.006922666976849239
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,512,fp8,0.0067498666544755295
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,256,fp8,0.006786133100589117
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,128,fp8,0.006565333157777785
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,64,fp8,0.006529066711664199
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,32,fp8,0.006606933226188024
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,128,65536,fp8,0.012161067128181458
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,16384,fp8,0.008225066463152566
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,12288,fp8,0.007890133062998453
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,10240,fp8,0.0072234672804673515
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,8192,fp8,0.007553066313266755
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,7168,fp8,0.007510399570067723
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,6144,fp8,0.007570133358240127
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,5120,fp8,0.007205332815647125
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,4096,fp8,0.007041066388289133
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,3584,fp8,0.007338666419188181
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,3072,fp8,0.007035733262697855
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,2560,fp8,0.007013333092133204
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,2048,fp8,0.006717866410811743
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,1536,fp8,0.007183999568223953
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,1024,fp8,0.007027200112740199
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,768,fp8,0.006916266183058421
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,512,fp8,0.006764799853165946
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,256,fp8,0.00652586668729782
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,128,fp8,0.006578133006890616
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,64,fp8,0.006753067175547281
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,32,fp8,0.006569600105285645
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,96,65536,fp8,0.011280001203219097
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,16384,fp8,0.007985066374142966
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,12288,fp8,0.007340800017118453
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,10240,fp8,0.007350399841864905
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,8192,fp8,0.007243733356396357
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,7168,fp8,0.007213866462310156
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,6144,fp8,0.006922666728496552
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,5120,fp8,0.007204266389211019
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,4096,fp8,0.006904533008734386
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,3584,fp8,0.007060266286134719
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,3072,fp8,0.0072213329374790185
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,2560,fp8,0.00688213308652242
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,2048,fp8,0.007102932780981064
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,1536,fp8,0.007274667173624039
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,1024,fp8,0.0067338670293490095
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,768,fp8,0.007006933043400447
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,512,fp8,0.006602667023738225
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,256,fp8,0.006505599866310755
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,128,fp8,0.0067797333002090465
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,64,fp8,0.006568533430496853
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,32,fp8,0.0067743999262650815
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,80,65536,fp8,0.01031040052572886
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,16384,fp8,0.0084394671022892
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,12288,fp8,0.007332266867160798
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,10240,fp8,0.007473066945870717
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,8192,fp8,0.007737600058317184
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,7168,fp8,0.007058133681615193
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,6144,fp8,0.007255466530720393
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,5120,fp8,0.006924799581368764
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,4096,fp8,0.006976000219583512
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,3584,fp8,0.007057067255179086
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,3072,fp8,0.006914132833480835
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,2560,fp8,0.006938666353623071
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,2048,fp8,0.007225600133339564
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,1536,fp8,0.00669760008653005
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,1024,fp8,0.00691626618305842
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,768,fp8,0.006677333265542984
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,512,fp8,0.006937599927186966
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,256,fp8,0.006704000383615494
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,128,fp8,0.00683840016523997
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,64,fp8,0.006550400207440059
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,32,fp8,0.006669867038726808
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,64,65536,fp8,0.011059200763702391
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,16384,fp8,0.00784426654378573
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,12288,fp8,0.007713066786527634
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,10240,fp8,0.007416532933712005
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,8192,fp8,0.007526399691899618
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,7168,fp8,0.007101866602897643
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,6144,fp8,0.007099732756614685
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,5120,fp8,0.007172266890605292
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,4096,fp8,0.0074549337228139246
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,3584,fp8,0.0071968004107475295
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,3072,fp8,0.007108266403277715
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,2560,fp8,0.006942932804425557
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,2048,fp8,0.007100799928108852
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,1536,fp8,0.007124267021814982
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,1024,fp8,0.007162666817506155
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,768,fp8,0.007038932790358861
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,512,fp8,0.006679466863473256
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,256,fp8,0.006910933057467143
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,128,fp8,0.006615466624498368
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,64,fp8,0.006750933577617009
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,32,fp8,0.006919466455777486
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,48,65536,fp8,0.009390933314959209
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,16384,fp8,0.007598933080832163
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,12288,fp8,0.007674666245778401
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,10240,fp8,0.007243733356396357
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,8192,fp8,0.007335466394821803
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,7168,fp8,0.006973866870005926
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,6144,fp8,0.007121066252390543
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,5120,fp8,0.007326933493216833
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,4096,fp8,0.006996266295512518
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,3584,fp8,0.007085866729418435
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,3072,fp8,0.007101866851250329
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,2560,fp8,0.007068800429503122
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,2048,fp8,0.007302399973074594
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,1536,fp8,0.0068522669374942785
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,1024,fp8,0.0069034665822982795
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,768,fp8,0.0069088004529476174
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,512,fp8,0.006719999760389328
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,256,fp8,0.006883200009663899
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,128,fp8,0.006712533285220465
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,64,fp8,0.006629333645105361
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,32,fp8,0.006909866134325663
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,32,65536,fp8,0.008099199334780377
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,16384,fp8,0.007156266272068023
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,12288,fp8,0.006994133690992992
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,10240,fp8,0.007099733253320058
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,8192,fp8,0.007174400488535564
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,7168,fp8,0.007055999586979549
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,6144,fp8,0.007329066594441732
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,5120,fp8,0.0068810666600863145
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,4096,fp8,0.007055999835332234
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,3584,fp8,0.00711253359913826
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,3072,fp8,0.006734933455785115
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,2560,fp8,0.0071338665982087456
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,2048,fp8,0.0067210664351781205
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,1536,fp8,0.006702933460474014
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,1024,fp8,0.006948266178369522
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,768,fp8,0.006676267087459564
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,512,fp8,0.0068629336853822075
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,256,fp8,0.006417066603899002
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,128,fp8,0.00675093357761701
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,64,fp8,0.006540800382693609
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,32,fp8,0.0068181338409582775
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,16,65536,fp8,0.007637333869934083
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,16384,fp8,0.00686720038453738
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,12288,fp8,0.007033600409825642
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,10240,fp8,0.006899199634790421
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,8192,fp8,0.006791466474533081
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,7168,fp8,0.0070207995673020685
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,6144,fp8,0.0068309334417184205
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,5120,fp8,0.006948266675074897
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,4096,fp8,0.006933333228031794
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,3584,fp8,0.006667732944091161
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,3072,fp8,0.006915200501680374
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,2560,fp8,0.0067605329056580855
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,2048,fp8,0.00678613359729449
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,1536,fp8,0.006604800124963125
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,1024,fp8,0.006849066664775213
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,768,fp8,0.006550399959087373
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,512,fp8,0.0064032000799973815
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,256,fp8,0.006717866907517115
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,128,fp8,0.006645333270231883
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,64,fp8,0.006838399668534596
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,32,fp8,0.006811733295520148
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,8,65536,fp8,0.007242666929960251
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,16384,fp8,0.006953600297371547
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,12288,fp8,0.006682666142781575
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,10240,fp8,0.006705066561698913
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,8192,fp8,0.006807466844717661
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,7168,fp8,0.006720000505447388
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,6144,fp8,0.006787199775377908
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,5120,fp8,0.006795732925335566
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,4096,fp8,0.006719999760389329
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,3584,fp8,0.007055999835332235
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,3072,fp8,0.006709333509206772
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,2560,fp8,0.00669760008653005
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,2048,fp8,0.006863999863465627
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,1536,fp8,0.006652799993753434
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,1024,fp8,0.006754133105278016
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,768,fp8,0.006929066280523936
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,512,fp8,0.006693333387374878
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,256,fp8,0.006929066280523936
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,128,fp8,0.0067327996095021565
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,64,fp8,0.006716800232728323
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,32,fp8,0.006877866884072622
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,4,65536,fp8,0.007178667187690734
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,16384,fp8,0.00692906677722931
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,12288,fp8,0.006670933465162913
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,10240,fp8,0.006754133105278016
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,8192,fp8,0.006636799623568853
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,7168,fp8,0.00692479982972145
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,6144,fp8,0.006996267040570576
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,5120,fp8,0.006882133334875105
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,4096,fp8,0.006882132838169734
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,3584,fp8,0.006650666644175848
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,3072,fp8,0.006743466357390086
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,2560,fp8,0.006884266436100007
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,2048,fp8,0.006568533678849539
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,1536,fp8,0.006947199751933416
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,1024,fp8,0.006834133466084798
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,768,fp8,0.006712533036867778
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,512,fp8,0.006922666480143865
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,256,fp8,0.006757333377997081
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,128,fp8,0.0068053337434927615
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,64,fp8,0.006623999774456024
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,32,fp8,0.006822400291760762
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,2,65536,fp8,0.0068725335101286564
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,16384,fp8,0.006568533182144166
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,12288,fp8,0.006844799717267354
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,10240,fp8,0.006627200047175089
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,8192,fp8,0.006786133348941803
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,7168,fp8,0.006809599945942561
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,6144,fp8,0.006488533566395441
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,5120,fp8,0.006783999751011531
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,4096,fp8,0.006459733098745346
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,3584,fp8,0.006665600339571634
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,3072,fp8,0.0070719997088114425
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,2560,fp8,0.00643199955423673
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,2048,fp8,0.006717867155869802
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,1536,fp8,0.00660799965262413
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,1024,fp8,0.0068597336610158295
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,768,fp8,0.006656000018119812
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,512,fp8,0.006824532896280289
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,256,fp8,0.0069514669477939595
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,128,fp8,0.00664213349421819
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,64,fp8,0.006833066294590633
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,32,fp8,0.00663679987192154
TRTLLM,1.2.0rc5,NVIDIA H200,compute_scale,torch_ops,1,65536,fp8,0.006783999999364217
