framework,version,device,op_name,kernel_source,m,k,quant_dtype,latency
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,16384,fp8,0.2770400047302246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,12288,fp8,0.2086858590443929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,10240,fp8,0.17450133959452313
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,8192,fp8,0.14035199483235677
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,7168,fp8,0.12321813106536865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,6144,fp8,0.10608853499094646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,5120,fp8,0.08928639888763427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,4096,fp8,0.07225600083669027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,3584,fp8,0.06359999974568685
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,3072,fp8,0.05491093397140503
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,2560,fp8,0.04633173147837321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,2048,fp8,0.03766186634699504
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,1536,fp8,0.020252799987792967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,1024,fp8,0.012702932953834534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,768,fp8,0.00995733340581258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,512,fp8,0.007380266487598419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,256,fp8,0.004654933512210846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,128,fp8,0.0037248000502586366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,64,fp8,0.0031914666295051576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,32,fp8,0.003047466774781545
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8192,65536,fp8,1.110630416870117
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,16384,fp8,0.1404981295267741
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,12288,fp8,0.10630293687184651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,10240,fp8,0.08910400072733561
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,8192,fp8,0.07232853571573893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,7168,fp8,0.06347413460413615
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,6144,fp8,0.05499306519826254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,5120,fp8,0.04637226661046346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,4096,fp8,0.0374069333076477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,3584,fp8,0.0302293340365092
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,3072,fp8,0.02031146685282389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,2560,fp8,0.015384533007939658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,2048,fp8,0.012654933333396911
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,1536,fp8,0.009910399715105694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,1024,fp8,0.007338666419188182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,768,fp8,0.005929600199063619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,512,fp8,0.00468800018231074
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,256,fp8,0.0036127999424934386
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,128,fp8,0.0032426667710145317
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,64,fp8,0.002930133293072383
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,32,fp8,0.0027082666754722597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4096,65536,fp8,0.553221321105957
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,16384,fp8,0.0721407969792684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,12288,fp8,0.054921599229176846
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,10240,fp8,0.04622933467229207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,8192,fp8,0.03775573174158732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,7168,fp8,0.03125440080960591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,6144,fp8,0.020304000377655028
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,5120,fp8,0.015498666961987814
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,4096,fp8,0.012634666760762534
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,3584,fp8,0.01123306651910146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,3072,fp8,0.009998933474222819
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,2560,fp8,0.008570667107899983
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,2048,fp8,0.0073183998465538025
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,1536,fp8,0.00592853327592214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,1024,fp8,0.004701866706212362
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,768,fp8,0.004153600086768469
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,512,fp8,0.0036149332920710243
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,256,fp8,0.0032448001205921174
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,128,fp8,0.0029258665939172106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,64,fp8,0.0027797333896160126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,32,fp8,0.0026943999032179515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2048,65536,fp8,0.2772063891092936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,16384,fp8,0.03778666655222575
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,12288,fp8,0.020434133211771646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,10240,fp8,0.015415466825167336
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,8192,fp8,0.012569600343704223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,7168,fp8,0.011348266402880352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,6144,fp8,0.009923199812571209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,5120,fp8,0.00865280032157898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,4096,fp8,0.007257600128650665
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,3584,fp8,0.006681600213050842
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,3072,fp8,0.005982933441797892
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,2560,fp8,0.005257600049177805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,2048,fp8,0.004714666803677877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,1536,fp8,0.004099199920892716
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,1024,fp8,0.0036938667297363283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,768,fp8,0.003370666752258936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,512,fp8,0.0032469332218170166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,256,fp8,0.0029365333418051405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,128,fp8,0.0027978666126728057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,64,fp8,0.0026890667776266735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,32,fp8,0.0027562665442625684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1024,65536,fp8,0.14007253646850587
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,16384,fp8,0.020569600661595664
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,12288,fp8,0.013914666573206582
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,10240,fp8,0.011986133456230164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,8192,fp8,0.00993066628774007
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,7168,fp8,0.00890773336092631
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,6144,fp8,0.008004266520341237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,5120,fp8,0.006915199756622315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,4096,fp8,0.005996799965699514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,3584,fp8,0.005334400137265523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,3072,fp8,0.0050122668345769245
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,2560,fp8,0.0045045331120491024
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,2048,fp8,0.004154666761557261
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,1536,fp8,0.0038176000118255613
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,1024,fp8,0.003505066782236099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,768,fp8,0.0032661333680152893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,512,fp8,0.0031541332602500914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,256,fp8,0.002794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,128,fp8,0.0028192001084486645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,64,fp8,0.002716800073782603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,32,fp8,0.0027903998891512555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,768,65536,fp8,0.1062165339787801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,16384,fp8,0.012588799993197123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,12288,fp8,0.010024533669153849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,10240,fp8,0.008642133076985676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,8192,fp8,0.007382399837176005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,7168,fp8,0.006622933348019918
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,6144,fp8,0.0060362666845321655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,5120,fp8,0.005286400020122528
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,4096,fp8,0.004726399978001913
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,3584,fp8,0.004355200131734212
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,3072,fp8,0.004193066557248434
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,2560,fp8,0.0038933334251244865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,2048,fp8,0.003737599899371465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,1536,fp8,0.0034314667185147605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,1024,fp8,0.0032821332414944967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,768,fp8,0.003094399968783061
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,512,fp8,0.0029674666623274487
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,256,fp8,0.002828799933195114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,128,fp8,0.00271573339899381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,64,fp8,0.0027978666126728057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,32,fp8,0.0027082666754722597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,512,65536,fp8,0.0720970630645752
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,16384,fp8,0.009959466258684794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,12288,fp8,0.008062933385372163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,10240,fp8,0.007032533486684163
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,8192,fp8,0.005962666869163513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,7168,fp8,0.005469866593678792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,6144,fp8,0.004972800115744273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,5120,fp8,0.0046304002404212955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,4096,fp8,0.004120533416668574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,3584,fp8,0.004070399949947992
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,3072,fp8,0.00383786658445994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,2560,fp8,0.0036490666369597114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,2048,fp8,0.003525333354870478
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,1536,fp8,0.003269333392381668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,1024,fp8,0.0031957333286603295
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,768,fp8,0.0030048000315825146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,512,fp8,0.002867199977238973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,256,fp8,0.0027306665976842242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,128,fp8,0.002799999962250392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,64,fp8,0.002713600049416224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,32,fp8,0.0027935999135176343
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,384,65536,fp8,0.05504533449808756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,16384,fp8,0.007386666536331177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,12288,fp8,0.005957333246866862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,10240,fp8,0.005364266534646353
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,8192,fp8,0.004655999938646952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,7168,fp8,0.004429866870244345
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,6144,fp8,0.004186666508515676
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,5120,fp8,0.003874133278926214
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,4096,fp8,0.00373333344856898
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,3584,fp8,0.003619199991226196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,3072,fp8,0.003517866631348928
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,2560,fp8,0.0033439998825391137
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,2048,fp8,0.003256533294916153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,1536,fp8,0.0030901332696278887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,1024,fp8,0.003014400104681651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,768,fp8,0.0027690666417280836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,512,fp8,0.0028181334336598715
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,256,fp8,0.002705066651105881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,128,fp8,0.0027669332921504974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,64,fp8,0.0026954665780067446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,32,fp8,0.0027882667879263563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,256,65536,fp8,0.03770560026168823
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,16384,fp8,0.005931733548641205
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,12288,fp8,0.005029333134492239
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,10240,fp8,0.004515199859937032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,8192,fp8,0.004164266586303711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,7168,fp8,0.003949866692225138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,6144,fp8,0.0038965334494908653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,5120,fp8,0.0036298667391141256
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,4096,fp8,0.0034730667869249977
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,3584,fp8,0.0033503999312718712
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,3072,fp8,0.0033173332611719764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,2560,fp8,0.003134933362404505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,2048,fp8,0.0031615999837716425
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,1536,fp8,0.002963199963172277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,1024,fp8,0.002850133428970973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,768,fp8,0.002734933296839396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,512,fp8,0.002794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,256,fp8,0.002757333219051361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,128,fp8,0.0026848000784715016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,64,fp8,0.002784000088771184
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,32,fp8,0.0026869334280490874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,192,65536,fp8,0.019853866100311278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,16384,fp8,0.005285333096981049
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,12288,fp8,0.004612266520659129
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,10240,fp8,0.004216533402601878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,8192,fp8,0.003861333429813385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,7168,fp8,0.0038624001046021783
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,6144,fp8,0.003640533238649368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,5120,fp8,0.0035349334279696143
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,4096,fp8,0.0033183999359607695
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,3584,fp8,0.00359253336985906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,3072,fp8,0.0031466667850812277
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,2560,fp8,0.00317546675602595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,2048,fp8,0.00306986669699351
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,1536,fp8,0.0028533334533373516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,1024,fp8,0.0028533334533373516
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,768,fp8,0.0027253332237402597
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,512,fp8,0.002776533365249634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,256,fp8,0.0026911998788515727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,128,fp8,0.0027722666660944624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,64,fp8,0.002696533252795537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,32,fp8,0.002762666592995326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,160,65536,fp8,0.01520853340625763
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,16384,fp8,0.0047082667549451195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,12288,fp8,0.004089600096146265
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,10240,fp8,0.003939199944337209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,8192,fp8,0.0036320000886917113
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,7168,fp8,0.0036501333117485045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,6144,fp8,0.0034133332471052804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,5120,fp8,0.003320533285538355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,4096,fp8,0.003260799994071325
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,3584,fp8,0.003125333289305369
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,3072,fp8,0.0031498665610949195
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,2560,fp8,0.0029834667841593427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,2048,fp8,0.0030218665798505146
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,1536,fp8,0.0027402666707833606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,1024,fp8,0.0028064000109831494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,768,fp8,0.0027562665442625684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,512,fp8,0.0028031999866167706
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,256,fp8,0.0027200000981489818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,128,fp8,0.00279573326309522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,64,fp8,0.0027274665733178455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,32,fp8,0.00279573326309522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,128,65536,fp8,0.012621866663297019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,16384,fp8,0.004192000130812327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,12288,fp8,0.003915733347336451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,10240,fp8,0.0036373332142829893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,8192,fp8,0.003502933432658514
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,7168,fp8,0.0034122665723164878
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,6144,fp8,0.0033482665816942847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,5120,fp8,0.0031839999059836066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,4096,fp8,0.0031786667803923286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,3584,fp8,0.003045333425203959
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,3072,fp8,0.0030730667213598887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,2560,fp8,0.002880000074704488
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,2048,fp8,0.0028607999285062153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,1536,fp8,0.002752000093460083
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,1024,fp8,0.0028234665592511495
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,768,fp8,0.0027317332724730173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,512,fp8,0.0027776000400384264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,256,fp8,0.0027189334233601887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,128,fp8,0.0027200000981489818
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,64,fp8,0.00277866671482722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,32,fp8,0.0027104000250498454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,96,65536,fp8,0.010040533542633057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,16384,fp8,0.0038975998759269714
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,12288,fp8,0.0037205333511034647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,10240,fp8,0.003453866640726725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,8192,fp8,0.003437866767247518
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,7168,fp8,0.003234133372704188
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,6144,fp8,0.003256533294916153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,5120,fp8,0.00311253344019254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,4096,fp8,0.0031146667897701263
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,3584,fp8,0.0029834667841593427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,3072,fp8,0.0029813334345817565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,2560,fp8,0.0028064000109831494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,2048,fp8,0.0028522667785485585
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,1536,fp8,0.002833066632350286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,1024,fp8,0.002734933296839396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,768,fp8,0.0028138667345046996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,512,fp8,0.002700799951950709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,256,fp8,0.0027797333896160126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,128,fp8,0.0027093333502610523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,64,fp8,0.0027893332143624624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,32,fp8,0.002716800073782603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,80,65536,fp8,0.008665600419044494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,16384,fp8,0.0036458666125933326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,12288,fp8,0.003509333233038584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,10240,fp8,0.0033674667278925574
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,8192,fp8,0.003271466741959254
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,7168,fp8,0.003151999910672506
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,6144,fp8,0.0031669333577156065
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,5120,fp8,0.0030303999781608583
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,4096,fp8,0.0029653333127498626
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,3584,fp8,0.0029098667204380036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,3072,fp8,0.0027978666126728057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,2560,fp8,0.0028362666567166646
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,2048,fp8,0.0027562665442625684
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,1536,fp8,0.0028170667588710784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,1024,fp8,0.0027221334477265675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,768,fp8,0.0027722666660944624
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,512,fp8,0.002713600049416224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,256,fp8,0.0027744000156720476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,128,fp8,0.0026752000053723653
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,64,fp8,0.0027669332921504974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,32,fp8,0.002666666607062022
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,64,65536,fp8,0.007346133391062419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,16384,fp8,0.0033952000240484873
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,12288,fp8,0.0033290666838486993
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,10240,fp8,0.003230933348337809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,8192,fp8,0.003070933371782303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,7168,fp8,0.0030965333183606463
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,6144,fp8,0.0029578665892283124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,5120,fp8,0.0029258665939172106
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,4096,fp8,0.002762666592995326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,3584,fp8,0.002833066632350286
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,3072,fp8,0.0027295999228954316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,2560,fp8,0.0028042666614055633
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,2048,fp8,0.0027061333258946735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,1536,fp8,0.002791466563940048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,1024,fp8,0.0026922665536403658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,768,fp8,0.002776533365249634
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,512,fp8,0.0026837334036827086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,256,fp8,0.002757333219051361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,128,fp8,0.0026837334036827086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,64,fp8,0.0026858667532602947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,32,fp8,0.002762666592995326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,48,65536,fp8,0.00594346672296524
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,16384,fp8,0.0032586666444937387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,12288,fp8,0.0030730667213598887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,10240,fp8,0.0031178665657838186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,8192,fp8,0.0029365333418051405
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,7168,fp8,0.002903466671705246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,6144,fp8,0.002841600030660629
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,5120,fp8,0.002739199995994568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,4096,fp8,0.0028010666370391845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,3584,fp8,0.002735999971628189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,3072,fp8,0.002808533360560735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,2560,fp8,0.002700799951950709
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,2048,fp8,0.0027776000400384264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,1536,fp8,0.0026848000784715016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,1024,fp8,0.0026869334280490874
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,768,fp8,0.00277866671482722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,512,fp8,0.0026837334036827086
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,256,fp8,0.00276053324341774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,128,fp8,0.0026954665780067446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,64,fp8,0.002755200117826462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,32,fp8,0.002681600054105123
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,32,65536,fp8,0.004683733483155568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,16384,fp8,0.0029440000653266907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,12288,fp8,0.0028309332827727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,10240,fp8,0.002739199995994568
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,8192,fp8,0.0028064000109831494
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,7168,fp8,0.0027306665976842242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,6144,fp8,0.002820266783237457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,5120,fp8,0.0026954665780067446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,4096,fp8,0.002773333340883255
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,3584,fp8,0.0027744000156720476
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,3072,fp8,0.0027061333258946735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,2560,fp8,0.002771199991305669
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,2048,fp8,0.0026848000784715016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,1536,fp8,0.0027701333165168762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,1024,fp8,0.0026858667532602947
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,768,fp8,0.0027690666417280836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,512,fp8,0.0027146667242050173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,256,fp8,0.002794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,128,fp8,0.0027189334233601887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,64,fp8,0.002796799937884013
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,32,fp8,0.002713600049416224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,16,65536,fp8,0.003737599899371465
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,16384,fp8,0.0027285332481066385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,12288,fp8,0.002820266783237457
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,10240,fp8,0.0027306665976842242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,8192,fp8,0.0027317332724730173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,7168,fp8,0.002799999962250392
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,6144,fp8,0.0027189334233601887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,5120,fp8,0.0028223998844623564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,4096,fp8,0.0027317332724730173
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,3584,fp8,0.002780800064404805
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,3072,fp8,0.0027114666998386385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,2560,fp8,0.002792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,2048,fp8,0.0027061333258946735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,1536,fp8,0.0028021333118279776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,1024,fp8,0.0027306665976842242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,768,fp8,0.002794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,512,fp8,0.0027114666998386385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,256,fp8,0.0027818667391935987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,128,fp8,0.002717866748571396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,64,fp8,0.0027989332874615988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,32,fp8,0.0027829334139823914
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,8,65536,fp8,0.003222399950027466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,16384,fp8,0.002794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,12288,fp8,0.00271573339899381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,10240,fp8,0.0028010666370391845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,8192,fp8,0.0027232001225153605
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,7168,fp8,0.002794666588306427
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,6144,fp8,0.0027221334477265675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,5120,fp8,0.0027221334477265675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,4096,fp8,0.003070933371782303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,3584,fp8,0.002705066651105881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,3072,fp8,0.002792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,2560,fp8,0.0027114666998386385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,2048,fp8,0.002791466563940048
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,1536,fp8,0.0027061333258946735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,1024,fp8,0.0027989332874615988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,768,fp8,0.0028053333361943563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,512,fp8,0.002721066772937775
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,256,fp8,0.002775466690460841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,128,fp8,0.002712533374627431
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,64,fp8,0.002792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,32,fp8,0.002716800073782603
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,4,65536,fp8,0.003053866575161616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,16384,fp8,0.0028170667588710784
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,12288,fp8,0.0027882667879263563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,10240,fp8,0.0026986666023731233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,8192,fp8,0.002792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,7168,fp8,0.0027295999228954316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,6144,fp8,0.0027776000400384264
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,5120,fp8,0.0027104000250498454
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,4096,fp8,0.0028010666370391845
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,3584,fp8,0.0027221334477265675
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,3072,fp8,0.0027242665489514667
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,2560,fp8,0.0027882667879263563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,2048,fp8,0.0026954665780067446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,1536,fp8,0.00278613343834877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,1024,fp8,0.0027274665733178455
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,768,fp8,0.002792533238728841
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,512,fp8,0.002705066651105881
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,256,fp8,0.00278613343834877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,128,fp8,0.0026762666801611584
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,64,fp8,0.0027690666417280836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,32,fp8,0.002665599932273229
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,2,65536,fp8,0.002810666710138321
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,16384,fp8,0.002682666728893916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,12288,fp8,0.00276053324341774
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,10240,fp8,0.002674133330583572
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,8192,fp8,0.0027701333165168762
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,7168,fp8,0.0027530667682488757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,6144,fp8,0.0027093333502610523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,5120,fp8,0.0027690666417280836
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,4096,fp8,0.0026805333793163298
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,3584,fp8,0.0027530667682488757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,3072,fp8,0.0026848000784715016
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,2560,fp8,0.002765866617361705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,2048,fp8,0.002677333354949951
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,1536,fp8,0.0027583998938401537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,1024,fp8,0.0026954665780067446
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,768,fp8,0.0027583998938401537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,512,fp8,0.0026986666023731233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,256,fp8,0.0027818667391935987
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,128,fp8,0.0026922665536403658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,64,fp8,0.0027530667682488757
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,32,fp8,0.0026890667776266735
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,scale_matrix,torch_ops,1,65536,fp8,0.0027061333258946735
