framework,version,device,op_name,kernel_source,m,k,quant_dtype,latency
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,16384,fp8,0.2142837524414063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,12288,fp8,0.16591358184814456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,10240,fp8,0.1370954513549805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,8192,fp8,0.11135892868041994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,7168,fp8,0.09856853485107422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,6144,fp8,0.08529066244761146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,5120,fp8,0.0714037338892619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,4096,fp8,0.05756159623463948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,3584,fp8,0.050680538018544524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,3072,fp8,0.04258560339609782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,2560,fp8,0.034905600547790516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,2048,fp8,0.024407466252644866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,1536,fp8,0.022703999280929567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,1024,fp8,0.012999466061592104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,768,fp8,0.011239467064539592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,512,fp8,0.00994986742734909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,256,fp8,0.008671999971071879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,128,fp8,0.007815466324488322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,64,fp8,0.007601066678762435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,32,fp8,0.006740266581376393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,65536,fp8,0.8329429626464844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,16384,fp8,0.11118292808532718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,12288,fp8,0.0853610674540202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,10240,fp8,0.07165333429972333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,8192,fp8,0.05807253519694011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,7168,fp8,0.051036794980367034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,6144,fp8,0.043369603157043454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,5120,fp8,0.03523093064626057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,4096,fp8,0.025056000550587973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,3584,fp8,0.022252798080444332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,3072,fp8,0.02213866710662842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,2560,fp8,0.017869866887728375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,2048,fp8,0.012823466459910076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,1536,fp8,0.011683200796445211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,1024,fp8,0.00977066606283188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,768,fp8,0.009176532924175263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,512,fp8,0.008322133123874665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,256,fp8,0.007609600077072779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,128,fp8,0.007602133353551229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,64,fp8,0.007257600128650666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,32,fp8,0.007189332942167917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,65536,fp8,0.41961917877197275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,16384,fp8,0.05774827003479002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,12288,fp8,0.04255146980285644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,10240,fp8,0.03440746863683065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,8192,fp8,0.024496002991994226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,7168,fp8,0.02144960165023804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,6144,fp8,0.02269973357518514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,5120,fp8,0.018637867768605553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,4096,fp8,0.01259733339150747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,3584,fp8,0.01187733312447866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,3072,fp8,0.011134933431943257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,2560,fp8,0.01030186613400777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,2048,fp8,0.009714133540789285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,1536,fp8,0.009138133625189462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,1024,fp8,0.008315733075141907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,768,fp8,0.008020266890525818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,512,fp8,0.008038400361935299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,256,fp8,0.007093332956234613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,128,fp8,0.007330133517583212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,64,fp8,0.007038933038711548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,32,fp8,0.007242666433254877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,65536,fp8,0.21469227472941077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,16384,fp8,0.02455146710077922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,12288,fp8,0.022082134087880456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,10240,fp8,0.018038400014241538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,8192,fp8,0.013083732128143311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,7168,fp8,0.01184320052464803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,6144,fp8,0.011041067043940228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,5120,fp8,0.010162133971850078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,4096,fp8,0.009798400600751243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,3584,fp8,0.0089738667011261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,3072,fp8,0.008888533711433411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,2560,fp8,0.008779733876387277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,2048,fp8,0.008274133006731669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,1536,fp8,0.008041599641243617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,1024,fp8,0.007877332965532939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,768,fp8,0.007377066711584727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,512,fp8,0.007436799506346384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,256,fp8,0.007233067105213801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,128,fp8,0.00690880020459493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,64,fp8,0.007116800049940745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,32,fp8,0.006793599824110668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,65536,fp8,0.11145919164021809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,16384,fp8,0.022695467869440714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,12288,fp8,0.015061334768931072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,10240,fp8,0.012531200051307678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,8192,fp8,0.010856533050537109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,7168,fp8,0.010853333274523418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,6144,fp8,0.01015999913215637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,5120,fp8,0.010099199910958609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,4096,fp8,0.008863999942938485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,3584,fp8,0.008874666194121044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,3072,fp8,0.008423466483751934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,2560,fp8,0.008918399612108866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,2048,fp8,0.008206933736801147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,1536,fp8,0.007905066261688869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,1024,fp8,0.007485866794983546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,768,fp8,0.007809066772460937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,512,fp8,0.007130666325489679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,256,fp8,0.0073535998662312835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,128,fp8,0.00706986611088117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,64,fp8,0.0073536003629366565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,32,fp8,0.0070421335597833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,65536,fp8,0.08559679985046387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,16384,fp8,0.013252267241477968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,12288,fp8,0.01114026606082916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,10240,fp8,0.010610133409500124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,8192,fp8,0.009596799314022067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,7168,fp8,0.009562666217486065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,6144,fp8,0.009572266042232512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,5120,fp8,0.008761599659919739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,4096,fp8,0.00840106705824534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,3584,fp8,0.008901333808898926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,3072,fp8,0.008730666339397428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,2560,fp8,0.007830400516589483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,2048,fp8,0.0077141334613164275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,1536,fp8,0.007796266923348108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,1024,fp8,0.007464533795913061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,768,fp8,0.007576533655325571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,512,fp8,0.007338666170835495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,256,fp8,0.00701653336485227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,128,fp8,0.007138132800658543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,64,fp8,0.0069322663048903155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,32,fp8,0.006957866748174032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,65536,fp8,0.05770667394002278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,16384,fp8,0.011256533861160278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,12288,fp8,0.00989333341519038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,10240,fp8,0.009266133606433868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,8192,fp8,0.009344000120957693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,7168,fp8,0.008805333574612934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,6144,fp8,0.008720000088214875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,5120,fp8,0.008369066814581553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,4096,fp8,0.007994666447242102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,3584,fp8,0.007883733759323757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,3072,fp8,0.007677866270144782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,2560,fp8,0.008104532957077026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,2048,fp8,0.007495466371377309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,1536,fp8,0.007807999600966772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,1024,fp8,0.007299199948708215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,768,fp8,0.0076789334416389465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,512,fp8,0.007317332923412322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,256,fp8,0.0072671997050444284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,128,fp8,0.006978133072455725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,64,fp8,0.007016532868146897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,32,fp8,0.006753066678841908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,65536,fp8,0.04285759925842285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,16384,fp8,0.009701333940029144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,12288,fp8,0.009246933460235597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,10240,fp8,0.008669866621494294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,8192,fp8,0.009533866743246714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,7168,fp8,0.00804480065902074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,6144,fp8,0.007850666840871174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,5120,fp8,0.008768000453710557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,4096,fp8,0.007644800593455633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,3584,fp8,0.007913599659999212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,3072,fp8,0.007484799871842066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,2560,fp8,0.007832532872756322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,2048,fp8,0.007356800387303035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,1536,fp8,0.0074784000714619965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,1024,fp8,0.007123199850320816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,768,fp8,0.007313066224257152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,512,fp8,0.007020800312360128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,256,fp8,0.0074282668530941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,128,fp8,0.006861867010593416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,64,fp8,0.006949333101511001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,32,fp8,0.006760532905658087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,65536,fp8,0.024581333001454667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,16384,fp8,0.009997866551081338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,12288,fp8,0.008262399832407633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,10240,fp8,0.008533333241939545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,8192,fp8,0.007944533228874208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,7168,fp8,0.007886933783690136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,6144,fp8,0.008594133456548053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,5120,fp8,0.008712533613046011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,4096,fp8,0.007278933127721152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,3584,fp8,0.00760000025232633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,3072,fp8,0.0075477329393227885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,2560,fp8,0.007384532938400904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,2048,fp8,0.0071690671145915996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,1536,fp8,0.007362133016188941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,1024,fp8,0.006974933048089344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,768,fp8,0.007100800424814224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,512,fp8,0.006850133339564007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,256,fp8,0.0069322668015956886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,128,fp8,0.007105066627264024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,64,fp8,0.006659200042486191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,32,fp8,0.007024000088373819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,65536,fp8,0.023160533110300703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,16384,fp8,0.008474666873613992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,12288,fp8,0.009114666779836019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,10240,fp8,0.00820160061120987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,8192,fp8,0.00812479977806409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,7168,fp8,0.00787946656346321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,6144,fp8,0.007699199765920641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,5120,fp8,0.007884800185759861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,4096,fp8,0.007608533153931301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,3584,fp8,0.00738453318675359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,3072,fp8,0.0075103998184204105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,2560,fp8,0.007133866846561433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,2048,fp8,0.0070581331849098195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,1536,fp8,0.007576533406972886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,1024,fp8,0.00694186637798945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,768,fp8,0.00714560051759084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,512,fp8,0.007045332839091619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,256,fp8,0.007173333565394083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,128,fp8,0.006789333621660868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,64,fp8,0.0069301334520181015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,32,fp8,0.0067114666104316715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,65536,fp8,0.018419198195139562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,16384,fp8,0.008266666531562807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,12288,fp8,0.00780053287744522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,10240,fp8,0.007591466854015986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,8192,fp8,0.007790933301051459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,7168,fp8,0.007338666667540868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,6144,fp8,0.007562666883071263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,5120,fp8,0.007460266600052515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,4096,fp8,0.007420799881219865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,3584,fp8,0.0072885329524676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,3072,fp8,0.007098666330178578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,2560,fp8,0.007502933094898861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,2048,fp8,0.007197866340478261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,1536,fp8,0.007245866705973943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,1024,fp8,0.00691413308183352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,768,fp8,0.007386666536331176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,512,fp8,0.007193600138028463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,256,fp8,0.00716586709022522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,128,fp8,0.006808533022801081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,64,fp8,0.0071157336235046385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,32,fp8,0.006838399668534597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,65536,fp8,0.013112534085909525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,16384,fp8,0.008077867329120636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,12288,fp8,0.007911467055479685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,10240,fp8,0.007956266403198242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,8192,fp8,0.007533866415421168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,7168,fp8,0.007674666245778401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,6144,fp8,0.00783893366654714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,5120,fp8,0.007619199653466543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,4096,fp8,0.007145600020885468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,3584,fp8,0.007548799862464269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,3072,fp8,0.007400532811880111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,2560,fp8,0.007336533069610596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,2048,fp8,0.0069877333939075475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,1536,fp8,0.007204266140858333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,1024,fp8,0.007125333696603774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,768,fp8,0.007337599744399388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,512,fp8,0.007055999835332235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,256,fp8,0.007096532980600992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,128,fp8,0.0069930667678515124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,64,fp8,0.0068128004670143134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,32,fp8,0.006985599547624589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,65536,fp8,0.010700801014900205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,16384,fp8,0.007832533617814382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,12288,fp8,0.00810346653064092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,10240,fp8,0.00789226641257604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,8192,fp8,0.007304533074299495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,7168,fp8,0.007579733679691951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,6144,fp8,0.007170133292675019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,5120,fp8,0.007463466624418894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,4096,fp8,0.00729493349790573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,3584,fp8,0.007565866162379583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,3072,fp8,0.007102933277686437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,2560,fp8,0.007221333434184393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,2048,fp8,0.007127466797828674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,1536,fp8,0.007131733000278473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,1024,fp8,0.007373866438865662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,768,fp8,0.0070144002636273696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,512,fp8,0.007352533191442489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,256,fp8,0.006793600072463353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,128,fp8,0.006966400394837061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,64,fp8,0.006835200140873592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,32,fp8,0.007067733009656269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,65536,fp8,0.010443734129269917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,16384,fp8,0.007794133822123211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,12288,fp8,0.0073397333423296605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,10240,fp8,0.007702399541934332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,8192,fp8,0.007369599739710489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,7168,fp8,0.007465599725643793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,6144,fp8,0.007156266272068024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,5120,fp8,0.0074378666778405515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,4096,fp8,0.007587200154860814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,3584,fp8,0.007143466174602509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,3072,fp8,0.007289599627256393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,2560,fp8,0.0070816000302632645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,2048,fp8,0.00760106717546781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,1536,fp8,0.0071957334876060495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,1024,fp8,0.007293866326411564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,768,fp8,0.007112533599138259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,512,fp8,0.007126400371392568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,256,fp8,0.0067200002570947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,128,fp8,0.007006933043400448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,64,fp8,0.006747733304897944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,32,fp8,0.0069034665822982795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,65536,fp8,0.009700265526771543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,16384,fp8,0.007709866265455881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,12288,fp8,0.0075925327837467185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,10240,fp8,0.007303466647863388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,8192,fp8,0.007336533814668655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,7168,fp8,0.007108266403277715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,6144,fp8,0.0072501336534818005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,5120,fp8,0.007180799792210261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,4096,fp8,0.007225599884986877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,3584,fp8,0.007061332960923513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,3072,fp8,0.007218133409818014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,2560,fp8,0.007189333687225977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,2048,fp8,0.007365333537260692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,1536,fp8,0.007012266417344412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,1024,fp8,0.007199999690055848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,768,fp8,0.007058133681615193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,512,fp8,0.007077333827813466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,256,fp8,0.006669866293668746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,128,fp8,0.006942933549483618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,64,fp8,0.006914133330186208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,32,fp8,0.006713600208361943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,65536,fp8,0.009483733276526135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,16384,fp8,0.007700266937414805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,12288,fp8,0.007682133714358013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,10240,fp8,0.007196800410747528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,8192,fp8,0.007326933493216834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,7168,fp8,0.0070495997865994775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,6144,fp8,0.0069653329749902095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,5120,fp8,0.007233066608508428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,4096,fp8,0.0070911998550097135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,3584,fp8,0.006971733520428339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,3072,fp8,0.007027199864387512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,2560,fp8,0.007139199723800024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,2048,fp8,0.006861866513888042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,1536,fp8,0.00711360052227974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,1024,fp8,0.007063466807206472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,768,fp8,0.006764800349871317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,512,fp8,0.006881066163380941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,256,fp8,0.006809599697589874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,128,fp8,0.006977066397666931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,64,fp8,0.0067071996629238136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,32,fp8,0.0069514664510885865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,65536,fp8,0.00828799953063329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,16384,fp8,0.007496533542871475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,12288,fp8,0.007013333588838577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,10240,fp8,0.007484799871842068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,8192,fp8,0.007065599660078684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,7168,fp8,0.007429333031177521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,6144,fp8,0.0071967994173367834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,5120,fp8,0.007245866705973943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,4096,fp8,0.006974933544794717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,3584,fp8,0.006920533627271652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,3072,fp8,0.007277866701285044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,2560,fp8,0.006786133348941802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,2048,fp8,0.007059199611345926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,1536,fp8,0.006822400043408076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,1024,fp8,0.007528533538182576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,768,fp8,0.006675199667612712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,512,fp8,0.006891733159621557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,256,fp8,0.006853333363930385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,128,fp8,0.006987733642260234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,64,fp8,0.006790399799744289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,32,fp8,0.006927999854087829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,65536,fp8,0.00764160007238388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,16384,fp8,0.007350399841864904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,12288,fp8,0.0072288001577059426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,10240,fp8,0.0072320001820723205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,8192,fp8,0.007227732986211777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,7168,fp8,0.007049599538246789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,6144,fp8,0.007102933277686438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,5120,fp8,0.0069535998006661735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,4096,fp8,0.007110400001207987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,3584,fp8,0.00696426679690679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,3072,fp8,0.007229866584142049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,2560,fp8,0.007013332843780517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,2048,fp8,0.007111466427644094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,1536,fp8,0.006728533158699672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,1024,fp8,0.007100800176461538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,768,fp8,0.0069152002533276885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,512,fp8,0.00706026703119278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,256,fp8,0.006772266576687496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,128,fp8,0.006993066767851512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,64,fp8,0.0067210669318834935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,32,fp8,0.006741333007812501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,65536,fp8,0.007719466586907706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,16384,fp8,0.006962133447329205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,12288,fp8,0.007396266361077626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,10240,fp8,0.007027199616034825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,8192,fp8,0.007243733356396357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,7168,fp8,0.006899199883143107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,6144,fp8,0.00720853383342425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,5120,fp8,0.007108266651630401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,4096,fp8,0.0069557333985964444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,3584,fp8,0.0071680001914501195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,3072,fp8,0.00672533338268598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,2560,fp8,0.007061333457628886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,2048,fp8,0.006887467205524444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,1536,fp8,0.007122133423884709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,1024,fp8,0.006829866270224253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,768,fp8,0.006520533810059228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,512,fp8,0.006975999474525451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,256,fp8,0.006781866401433946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,128,fp8,0.006945066899061203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,64,fp8,0.006726399809122085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,32,fp8,0.007008000214894612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,65536,fp8,0.007410133630037306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,16384,fp8,0.007145600269238154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,12288,fp8,0.007028266290823619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,10240,fp8,0.007014400511980058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,8192,fp8,0.0068010667959849044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,7168,fp8,0.007024000336726506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,6144,fp8,0.00673813372850418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,5120,fp8,0.007016533116499582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,4096,fp8,0.006935466825962067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,3584,fp8,0.007067733754714329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,3072,fp8,0.007088000078996023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,2560,fp8,0.006756266703208287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,2048,fp8,0.007038933287064233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,1536,fp8,0.006773333251476289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,1024,fp8,0.006986666222413382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,768,fp8,0.006777600447336833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,512,fp8,0.007038933535416922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,256,fp8,0.006790400048096974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,128,fp8,0.006873599936564764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,64,fp8,0.006730666508277258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,32,fp8,0.006932267049948374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,65536,fp8,0.007058132936557133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,16384,fp8,0.006980266670385996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,12288,fp8,0.006674133241176605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,10240,fp8,0.006947200000286102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,8192,fp8,0.006750933329264323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,7168,fp8,0.006729599585135777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,6144,fp8,0.006680533538262048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,5120,fp8,0.0067231997847557075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,4096,fp8,0.006945066650708517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,3584,fp8,0.006654933591683705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,3072,fp8,0.006853333115577698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,2560,fp8,0.006712533781925837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,2048,fp8,0.0068842666844526924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,1536,fp8,0.006741333256165186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,1024,fp8,0.0069450671474138885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,768,fp8,0.006721067180236179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,512,fp8,0.006866133213043213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,256,fp8,0.006654933343331019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,128,fp8,0.006872533510128658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,64,fp8,0.006666666517655054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,32,fp8,0.006844800462325415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,65536,fp8,0.007230933258930842
