framework,version,device,op_name,kernel_source,m,k,quant_dtype,latency
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,16384,fp8,0.2728810628255208
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,12288,fp8,0.20605546633402505
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,10240,fp8,0.17222399711608888
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,8192,fp8,0.13827412923177082
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,7168,fp8,0.12113173007965088
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,6144,fp8,0.10452053546905518
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,5120,fp8,0.0876032034556071
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,4096,fp8,0.0564789334932963
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,3584,fp8,0.03601173162460327
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,3072,fp8,0.029683200518290202
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,2560,fp8,0.02513386607170105
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,2048,fp8,0.02104960083961487
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,1536,fp8,0.016345600287119545
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,1024,fp8,0.012087466319402058
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,768,fp8,0.009973333279291789
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,512,fp8,0.007323733468850453
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,256,fp8,0.0050784001747767125
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,128,fp8,0.003916800022125244
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,64,fp8,0.0034506666163603462
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,32,fp8,0.003014400104681651
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8192,65536,fp8,4.903035481770834
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,16384,fp8,0.15182080268859863
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,12288,fp8,0.11267840067545573
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,10240,fp8,0.08778560161590576
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,8192,fp8,0.06312533219655356
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,7168,fp8,0.06843946774800619
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,6144,fp8,0.05869973500569662
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,5120,fp8,0.035306668281555174
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,4096,fp8,0.027137066920598345
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,3584,fp8,0.024331732590993246
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,3072,fp8,0.021471999088923135
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,2560,fp8,0.01853653391202291
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,2048,fp8,0.01530880033969879
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,1536,fp8,0.012796800335248312
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,1024,fp8,0.009553066889444987
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,768,fp8,0.007952000200748443
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,512,fp8,0.00631466656923294
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,256,fp8,0.004653866589069367
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,128,fp8,0.004106666644414266
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,64,fp8,0.003832533210515976
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,32,fp8,0.0030517332255840302
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4096,65536,fp8,4.355338541666667
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,16384,fp8,0.09538026650746664
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,12288,fp8,0.10442986488342285
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,10240,fp8,0.06709973017374674
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,8192,fp8,0.053904000918070474
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,7168,fp8,0.045689598719278975
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,6144,fp8,0.04009600083033244
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,5120,fp8,0.03381119966506958
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,4096,fp8,0.027679999669392902
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,3584,fp8,0.024868265787760416
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,3072,fp8,0.022426666816075642
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,2560,fp8,0.01939199964205424
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,2048,fp8,0.01660693287849426
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,1536,fp8,0.013667200009028116
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,1024,fp8,0.010445866982142131
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,768,fp8,0.009834667046864826
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,512,fp8,0.007507200042406718
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,256,fp8,0.006291200220584869
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,128,fp8,0.005485866467158
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,64,fp8,0.004458666841189066
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,32,fp8,0.00391146664818128
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2048,65536,fp8,1.3364757537841796
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,16384,fp8,0.06955946286519368
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,12288,fp8,0.04031039873758952
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,10240,fp8,0.03400213321050008
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,8192,fp8,0.028009599447250365
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,7168,fp8,0.025092266003290814
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,6144,fp8,0.022437334060668945
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,5120,fp8,0.019339734315872194
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,4096,fp8,0.016511999567349753
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,3584,fp8,0.01613653302192688
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,3072,fp8,0.013530666629473368
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,2560,fp8,0.012918399771054587
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,2048,fp8,0.010823466380437215
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,1536,fp8,0.009573333462079366
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,1024,fp8,0.0075445334116617845
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,768,fp8,0.007141333321730297
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,512,fp8,0.006297599772612255
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,256,fp8,0.0056970665852228795
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,128,fp8,0.004681600133577982
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,64,fp8,0.00383786658445994
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,32,fp8,0.0037994667887687682
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1024,65536,fp8,0.6888352076212565
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,16384,fp8,0.04771519899368286
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,12288,fp8,0.031027199824651082
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,10240,fp8,0.02759360074996948
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,8192,fp8,0.02221333384513855
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,7168,fp8,0.02105493346850077
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,6144,fp8,0.018645334243774413
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,5120,fp8,0.016523733735084534
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,4096,fp8,0.013750400145848593
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,3584,fp8,0.013017599781354269
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,3072,fp8,0.012039466698964437
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,2560,fp8,0.010798933108647664
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,2048,fp8,0.00957973301410675
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,1536,fp8,0.007962666451931
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,1024,fp8,0.007259733478228251
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,768,fp8,0.0065098668138186145
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,512,fp8,0.005892266829808554
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,256,fp8,0.004686933259169261
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,128,fp8,0.004249600072701773
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,64,fp8,0.003864533454179764
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,32,fp8,0.00344106654326121
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,768,65536,fp8,0.11894506613413494
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,16384,fp8,0.02977919975916545
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,12288,fp8,0.023492266734441124
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,10240,fp8,0.020706133047739664
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,8192,fp8,0.01737066706021627
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,7168,fp8,0.016901334126790367
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,6144,fp8,0.014129066467285156
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,5120,fp8,0.013272533814112345
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,4096,fp8,0.010809600353240967
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,3584,fp8,0.011246933539708456
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,3072,fp8,0.010385066270828247
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,2560,fp8,0.009161600470542907
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,2048,fp8,0.007928533355395
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,1536,fp8,0.007541333138942718
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,1024,fp8,0.006712533533573151
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,768,fp8,0.006243200103441874
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,512,fp8,0.005482666691144307
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,256,fp8,0.004735999802748362
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,128,fp8,0.004252799848715464
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,64,fp8,0.003841066608826319
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,32,fp8,0.0038719999293486277
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,512,65536,fp8,0.20257813135782876
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,16384,fp8,0.027188267310460406
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,12288,fp8,0.02074026664098104
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,10240,fp8,0.017548799514770508
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,8192,fp8,0.014512000481287637
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,7168,fp8,0.014074666301409402
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,6144,fp8,0.012250666817029316
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,5120,fp8,0.01120853324731191
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,4096,fp8,0.009991466999053955
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,3584,fp8,0.0091839998960495
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,3072,fp8,0.00840106705824534
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,2560,fp8,0.007527466615041096
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,2048,fp8,0.0079434668024381
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,1536,fp8,0.007114666700363159
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,1024,fp8,0.005899733304977417
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,768,fp8,0.005898666878541311
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,512,fp8,0.0050911997755368555
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,256,fp8,0.00425493319829305
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,128,fp8,0.004243200023969015
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,64,fp8,0.00386559988061587
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,32,fp8,0.0038474666575590765
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,384,65536,fp8,0.05915946563084921
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,16384,fp8,0.09277973175048829
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,12288,fp8,0.07455893357594809
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,10240,fp8,0.0686357339223226
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,8192,fp8,0.05634453296661377
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,7168,fp8,0.055387731393178305
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,6144,fp8,0.04995306730270386
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,5120,fp8,0.04421866734822591
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,4096,fp8,0.037028264999389646
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,3584,fp8,0.03663253386815389
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,3072,fp8,0.03622186581293742
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,2560,fp8,0.03335573275883992
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,2048,fp8,0.03067733248074849
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,1536,fp8,0.027204267183939618
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,1024,fp8,0.024659200509389242
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,768,fp8,0.02264960010846456
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,512,fp8,0.020309333006540933
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,256,fp8,0.01655466655890147
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,128,fp8,0.01573013365268707
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,64,fp8,0.014958932995796204
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,32,fp8,0.014920533696810404
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,256,65536,fp8,0.2972053209940592
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,16384,fp8,0.07451626459757486
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,12288,fp8,0.0627072016398112
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,10240,fp8,0.05532373189926147
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,8192,fp8,0.04992213249206543
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,7168,fp8,0.045661866664886475
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,6144,fp8,0.04073280096054077
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,5120,fp8,0.036602667967478436
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,4096,fp8,0.036210131645202634
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,3584,fp8,0.033538134892781575
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,3072,fp8,0.03289600014686585
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,2560,fp8,0.030403200785319013
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,2048,fp8,0.02718399961789449
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,1536,fp8,0.025967999299367266
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,1024,fp8,0.022670932610829673
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,768,fp8,0.020231467485427857
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,512,fp8,0.01898346741994222
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,256,fp8,0.015723733107248943
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,128,fp8,0.014907733599344889
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,64,fp8,0.014923733472824097
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,32,fp8,0.01448853313922882
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,192,65536,fp8,0.22951893806457518
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,16384,fp8,0.06827627023061117
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,12288,fp8,0.055460266272226964
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,10240,fp8,0.05164586702982584
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,8192,fp8,0.04434026479721069
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,7168,fp8,0.039885866641998294
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,6144,fp8,0.03623146613438924
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,5120,fp8,0.03661653200785319
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,4096,fp8,0.033367466926574704
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,3584,fp8,0.03210666577021281
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,3072,fp8,0.03038613398869832
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,2560,fp8,0.02726080020268758
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,2048,fp8,0.026601600646972656
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,1536,fp8,0.024259199698766075
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,1024,fp8,0.022601600488026938
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,768,fp8,0.019266132513682047
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,512,fp8,0.01648853321870168
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,256,fp8,0.015711999932924905
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,128,fp8,0.014907733599344889
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,64,fp8,0.014498133460680643
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,32,fp8,0.014516266187032065
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,160,65536,fp8,0.19511574109395344
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,16384,fp8,0.05628373225529989
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,12288,fp8,0.04973013401031494
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,10240,fp8,0.0444053332010905
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,8192,fp8,0.037037865320841475
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,7168,fp8,0.03647893269856771
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,6144,fp8,0.0366538683573405
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,5120,fp8,0.033326933781305954
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,4096,fp8,0.030738133192062377
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,3584,fp8,0.028823467095692952
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,3072,fp8,0.02720746596654256
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,2560,fp8,0.026719999313354493
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,2048,fp8,0.024327466885248818
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,1536,fp8,0.022699733575185142
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,1024,fp8,0.02064746618270874
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,768,fp8,0.01872640053431193
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,512,fp8,0.016539733608563742
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,256,fp8,0.015726932883262636
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,128,fp8,0.014696533481280008
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,64,fp8,0.014913066228230795
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,32,fp8,0.014473600188891092
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,128,65536,fp8,0.16113386154174805
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,16384,fp8,0.049985067049662275
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,12288,fp8,0.04041813214619954
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,10240,fp8,0.0366154670715332
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,8192,fp8,0.03662826617558797
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,7168,fp8,0.03361920118331909
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,6144,fp8,0.03287146687507629
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,5120,fp8,0.03006400068600972
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,4096,fp8,0.02719680070877075
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,3584,fp8,0.026728532711664837
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,3072,fp8,0.02595413327217102
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,2560,fp8,0.02427306572596232
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,2048,fp8,0.022691200176874794
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,1536,fp8,0.020566399892171225
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,1024,fp8,0.01894506613413493
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,768,fp8,0.016537599762280784
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,512,fp8,0.015736533204714458
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,256,fp8,0.01477120021979014
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,128,fp8,0.014891733725865683
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,64,fp8,0.01448853313922882
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,32,fp8,0.014518400033315023
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,96,65536,fp8,0.12744320233662923
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,16384,fp8,0.044130134582519534
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,12288,fp8,0.0366485317548116
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,10240,fp8,0.036690131823221846
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,8192,fp8,0.03330026666323344
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,7168,fp8,0.032237867514292404
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,6144,fp8,0.03007146716117859
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,5120,fp8,0.027265065908432008
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,4096,fp8,0.02653653422991435
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,3584,fp8,0.025156267484029132
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,3072,fp8,0.02430400053660075
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,2560,fp8,0.024256000916163124
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,2048,fp8,0.022657066583633423
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,1536,fp8,0.019181867440541588
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,1024,fp8,0.01653439998626709
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,768,fp8,0.016753067572911583
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,512,fp8,0.01572480003039042
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,256,fp8,0.014495999614397685
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,128,fp8,0.014917332927385965
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,64,fp8,0.014892799655596414
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,32,fp8,0.014494933684666953
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,80,65536,fp8,0.10975360075632731
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,16384,fp8,0.037360000610351565
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,12288,fp8,0.0366048018137614
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,10240,fp8,0.0333407998085022
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,8192,fp8,0.030730666716893514
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,7168,fp8,0.028445865710576373
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,6144,fp8,0.027195733785629273
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,5120,fp8,0.026613332827885944
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,4096,fp8,0.024411733945210776
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,3584,fp8,0.024331732590993246
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,3072,fp8,0.022900267442067464
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,2560,fp8,0.022626133759816487
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,2048,fp8,0.020567466815312706
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,1536,fp8,0.01899626652399699
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,1024,fp8,0.01648853321870168
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,768,fp8,0.01574720044930776
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,512,fp8,0.01574399967988332
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,256,fp8,0.014919466773668923
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,128,fp8,0.014843733112017313
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,64,fp8,0.014486400286356607
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,32,fp8,0.014917332927385965
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,64,65536,fp8,0.09239359696706137
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,16384,fp8,0.03621120055516561
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,12288,fp8,0.03293866713841756
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,10240,fp8,0.03012160062789917
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,8192,fp8,0.02760106722513835
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,7168,fp8,0.026789333422978717
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,6144,fp8,0.02571093241373698
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,5120,fp8,0.024410667022069295
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,4096,fp8,0.02267306645711263
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,3584,fp8,0.02255893349647522
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,3072,fp8,0.020660267273585
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,2560,fp8,0.019287467002868652
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,2048,fp8,0.01895893414815267
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,1536,fp8,0.01654293338457743
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,1024,fp8,0.015723733107248943
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,768,fp8,0.01570453345775604
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,512,fp8,0.014916266997655234
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,256,fp8,0.014919466773668923
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,128,fp8,0.014525866508483887
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,64,fp8,0.014910933375358582
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,32,fp8,0.014491732915242514
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,48,65536,fp8,0.0744874636332194
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,16384,fp8,0.030671999851862593
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,12288,fp8,0.027178666989008587
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,10240,fp8,0.026718932390213012
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,8192,fp8,0.024729599555333458
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,7168,fp8,0.024125866095225015
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,6144,fp8,0.022642133633295695
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,5120,fp8,0.02275200088818868
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,4096,fp8,0.02062293291091919
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,3584,fp8,0.019004799922307334
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,3072,fp8,0.019003732999165853
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,2560,fp8,0.0164874662955602
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,2048,fp8,0.016528000434239708
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,1536,fp8,0.015732266505559287
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,1024,fp8,0.015732266505559287
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,768,fp8,0.014900267124176025
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,512,fp8,0.014923733472824097
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,256,fp8,0.014501333236694336
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,128,fp8,0.014897066354751586
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,64,fp8,0.014486400286356607
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,32,fp8,0.014481066664059957
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,32,65536,fp8,0.05650026798248291
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,16384,fp8,0.024719999233881632
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,12288,fp8,0.0227018674214681
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,10240,fp8,0.02281173268953959
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,8192,fp8,0.020565332969029744
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,7168,fp8,0.01895786722501119
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,6144,fp8,0.01899519960085551
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,5120,fp8,0.0165503998597463
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,4096,fp8,0.0165503998597463
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,3584,fp8,0.01586666703224182
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,3072,fp8,0.015711999932924905
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,2560,fp8,0.015711999932924905
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,2048,fp8,0.015732266505559287
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,1536,fp8,0.014853333433469137
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,1024,fp8,0.014904533823331198
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,768,fp8,0.014904533823331198
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,512,fp8,0.014627200365066529
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,256,fp8,0.014849066734313965
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,128,fp8,0.014512000481287637
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,64,fp8,0.014487466216087342
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,32,fp8,0.014913066228230795
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,16,65536,fp8,0.03702720006306966
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,16384,fp8,0.020603734254837035
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,12288,fp8,0.018989866971969603
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,10240,fp8,0.016539733608563742
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,8192,fp8,0.016541866461435954
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,7168,fp8,0.015736533204714458
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,6144,fp8,0.015726932883262636
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,5120,fp8,0.01572480003039042
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,4096,fp8,0.015732266505559287
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,3584,fp8,0.015710933009783427
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,3072,fp8,0.01490239997704824
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,2560,fp8,0.014646400014559427
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,2048,fp8,0.014663466811180114
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,1536,fp8,0.014886400103569031
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,1024,fp8,0.014854400356610616
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,768,fp8,0.01454080045223236
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,512,fp8,0.014914133151372275
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,256,fp8,0.014493866761525472
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,128,fp8,0.0144896000623703
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,64,fp8,0.014906666676203408
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,32,fp8,0.014504533012708029
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,8,65536,fp8,0.030804266532262165
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,16384,fp8,0.016593066851298015
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,12288,fp8,0.01574186682701111
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,10240,fp8,0.015711999932924905
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,8192,fp8,0.01572053333123525
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,7168,fp8,0.015742933750152587
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,6144,fp8,0.014921599626541137
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,5120,fp8,0.015709867080052696
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,4096,fp8,0.014900267124176025
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,3584,fp8,0.014906666676203408
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,3072,fp8,0.014499200383822122
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,2560,fp8,0.014892799655596414
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,2048,fp8,0.014505599935849508
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,1536,fp8,0.014517333110173544
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,1024,fp8,0.014909866452217101
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,768,fp8,0.014516266187032065
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,512,fp8,0.014905599753061929
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,256,fp8,0.01490239997704824
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,128,fp8,0.014493866761525472
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,64,fp8,0.014507733782132468
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,32,fp8,0.014082133769989014
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,4,65536,fp8,0.02431466579437256
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,16384,fp8,0.015749333302179973
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,12288,fp8,0.014891733725865683
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,10240,fp8,0.01476800044377645
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,8192,fp8,0.014502400159835815
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,7168,fp8,0.014912000298500061
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,6144,fp8,0.014735999703407287
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,5120,fp8,0.014898133277893067
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,4096,fp8,0.01490133305390676
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,3584,fp8,0.014504533012708029
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,3072,fp8,0.014912000298500061
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,2560,fp8,0.014524799585342408
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,2048,fp8,0.014499200383822122
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,1536,fp8,0.01492586632569631
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,1024,fp8,0.014505599935849508
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,768,fp8,0.014896000425020853
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,512,fp8,0.014898133277893067
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,256,fp8,0.014509866635004679
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,128,fp8,0.014750933647155762
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,64,fp8,0.014030933380126953
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,32,fp8,0.014103466272354126
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,2,65536,fp8,0.02063360015551249
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,16384,fp8,0.014578133821487427
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,12288,fp8,0.014892799655596414
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,10240,fp8,0.014858667055765787
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,8192,fp8,0.014683733383814493
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,7168,fp8,0.014897066354751586
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,6144,fp8,0.014497066537539164
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,5120,fp8,0.014528000354766845
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,4096,fp8,0.014892799655596414
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,3584,fp8,0.014476799964904785
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,3072,fp8,0.014894933501879374
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,2560,fp8,0.014894933501879374
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,2048,fp8,0.014498133460680643
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,1536,fp8,0.014545067151387533
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,1024,fp8,0.01490239997704824
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,768,fp8,0.014868266383806863
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,512,fp8,0.014499200383822122
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,256,fp8,0.014501333236694336
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,128,fp8,0.01402239998181661
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,64,fp8,0.014082133769989014
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,32,fp8,0.014307199915250143
TRTLLM,1.0.0rc6,NVIDIA B200,scale_matrix,torch_ops,1,65536,fp8,0.016482133666674295
