framework,version,device,op_name,kernel_source,m,k,quant_dtype,latency
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,16384,fp8,0.08925867080688477
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,12288,fp8,0.06771626472473147
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,10240,fp8,0.057636276880900034
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,8192,fp8,0.04430400530497233
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,7168,fp8,0.038129067420959464
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,6144,fp8,0.030738123257954914
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,5120,fp8,0.01970345973968507
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,4096,fp8,0.02154453198115032
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,3584,fp8,0.027176535129547118
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,3072,fp8,0.021486934026082358
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,2560,fp8,0.017594665288925174
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,2048,fp8,0.014440530538558958
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,1536,fp8,0.014070399602254233
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,1024,fp8,0.012416001160939535
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,768,fp8,0.012321066856384278
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,512,fp8,0.011219199995199838
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,256,fp8,0.011221333344777427
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,128,fp8,0.010401067137718201
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,64,fp8,0.00900693361957868
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,32,fp8,0.009140266478061676
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8192,65536,fp8,0.0
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,16384,fp8,0.042670933405558276
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,12288,fp8,0.03613012631734211
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,10240,fp8,0.01931626796722412
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,8192,fp8,0.02521066665649413
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,7168,fp8,0.00538559754689534
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,6144,fp8,0.03292266527811686
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,5120,fp8,0.041031463940938324
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,4096,fp8,0.018456532557805386
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,3584,fp8,0.019026132424672448
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,3072,fp8,0.017161601781845098
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,2560,fp8,0.017124265432357788
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,2048,fp8,0.014506666858990987
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,1536,fp8,0.014485332369804378
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,1024,fp8,0.012885334094365439
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,768,fp8,0.013333333035310112
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,512,fp8,0.012795732915401458
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,256,fp8,0.012330667674541475
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,128,fp8,0.01060373286406199
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,64,fp8,0.009121067076921462
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,32,fp8,0.009419733037551245
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4096,65536,fp8,0.0
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,16384,fp8,0.0
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,12288,fp8,0.0520960013071696
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,10240,fp8,0.04318506717681886
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,8192,fp8,0.034447995821634936
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,7168,fp8,0.03221440315246581
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,6144,fp8,0.02829760313034057
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,5120,fp8,0.028072532018025717
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,4096,fp8,0.025203200181325273
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,3584,fp8,0.02650240262349447
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,3072,fp8,0.024924800793329878
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,2560,fp8,0.023473066091537476
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,2048,fp8,0.020578132073084517
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,1536,fp8,0.02158186634381612
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,1024,fp8,0.02069013317426046
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,768,fp8,0.018913066387176512
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,512,fp8,0.020043733716011047
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,256,fp8,0.016083199282487234
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,128,fp8,0.014079999923706055
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,64,fp8,0.014314666390419006
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,32,fp8,0.013894399752219516
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2048,65536,fp8,1.0138528188069662
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,16384,fp8,0.09658239682515463
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,12288,fp8,0.03132586479187011
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,10240,fp8,0.027353600660959883
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,8192,fp8,0.023669332265853882
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,7168,fp8,0.026619734366734822
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,6144,fp8,0.023364265759785965
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,5120,fp8,0.02345386544863383
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,4096,fp8,0.021400532126426692
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,3584,fp8,0.020726400613784793
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,3072,fp8,0.02237440049648285
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,2560,fp8,0.020678401986757917
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,2048,fp8,0.020167466004689535
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,1536,fp8,0.020125866929690046
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,1024,fp8,0.01861760020256043
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,768,fp8,0.01785386651754379
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,512,fp8,0.0160629337032636
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,256,fp8,0.01436160107453664
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,128,fp8,0.013999999562899271
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,64,fp8,0.014805333813031514
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,32,fp8,0.013799466192722319
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1024,65536,fp8,0.0
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,16384,fp8,0.08992319504419964
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,12288,fp8,0.0270197331905365
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,10240,fp8,0.026537601153055822
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,8192,fp8,0.025209599733352663
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,7168,fp8,0.02247466842333476
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,6144,fp8,0.022172800699869797
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,5120,fp8,0.021290664871533707
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,4096,fp8,0.021690664688746135
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,3584,fp8,0.023645864923795064
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,3072,fp8,0.020878932873407998
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,2560,fp8,0.01957226594289144
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,2048,fp8,0.019721600413322448
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,1536,fp8,0.01891840050617854
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,1024,fp8,0.01750186582406362
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,768,fp8,0.016203732788562776
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,512,fp8,0.015343999862670899
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,256,fp8,0.015166932841142018
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,128,fp8,0.014236800372600557
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,64,fp8,0.014149333288272222
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,32,fp8,0.01320746640364329
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,768,65536,fp8,0.1089664061864217
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,16384,fp8,0.023254398504892988
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,12288,fp8,0.027761067946751906
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,10240,fp8,0.023937066396077473
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,8192,fp8,0.021892267465591427
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,7168,fp8,0.02214399973551432
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,6144,fp8,0.02393813530604045
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,5120,fp8,0.023425064484278363
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,4096,fp8,0.022098133961359664
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,3584,fp8,0.020623999834060668
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,3072,fp8,0.019897600015004475
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,2560,fp8,0.020310398936271665
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,2048,fp8,0.019023999571800232
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,1536,fp8,0.017446400225162504
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,1024,fp8,0.016261333723862965
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,768,fp8,0.016276267170906068
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,512,fp8,0.01696639955043793
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,256,fp8,0.01420266677935918
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,128,fp8,0.014240000148614249
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,64,fp8,0.014451199521621067
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,32,fp8,0.01435519978404045
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,512,65536,fp8,0.0
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,16384,fp8,0.05855252941449483
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,12288,fp8,0.022438398996988934
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,10240,fp8,0.022905600070953366
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,8192,fp8,0.024446934461593628
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,7168,fp8,0.022427734732627873
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,6144,fp8,0.022375467419624328
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,5120,fp8,0.021217067042986554
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,4096,fp8,0.020978132883707683
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,3584,fp8,0.02051413357257843
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,3072,fp8,0.021275733908017475
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,2560,fp8,0.019426133235295615
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,2048,fp8,0.017238398889700572
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,1536,fp8,0.017182934284210204
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,1024,fp8,0.01573119958241781
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,768,fp8,0.014988799393177031
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,512,fp8,0.015581866602102916
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,256,fp8,0.01482879916826884
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,128,fp8,0.014650665720303853
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,64,fp8,0.014403199901183446
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,32,fp8,0.014637866864601771
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,384,65536,fp8,0.06182079712549845
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,16384,fp8,0.0
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,12288,fp8,0.10901973247528074
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,10240,fp8,0.10414613087972006
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,8192,fp8,0.0995861291885376
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,7168,fp8,0.09861867030461631
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,6144,fp8,0.09555412530899049
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,5120,fp8,0.09377919832865397
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,4096,fp8,0.09494080543518067
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,3584,fp8,0.08765120108922324
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,3072,fp8,0.08370986382166544
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,2560,fp8,0.07962239980697632
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,2048,fp8,0.07479360302289328
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,1536,fp8,0.07255146503448485
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,1024,fp8,0.06785173018773397
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,768,fp8,0.06851306557655334
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,512,fp8,0.06575466394424438
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,256,fp8,0.06362026433149973
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,128,fp8,0.06267306903998057
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,64,fp8,0.06150400340557098
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,32,fp8,0.06143253147602082
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,256,65536,fp8,0.1659189542134603
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,16384,fp8,0.10885759989420572
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,12288,fp8,0.11005759239196777
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,10240,fp8,0.09882773955663045
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,8192,fp8,0.10330133438110353
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,7168,fp8,0.09646933476130168
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,6144,fp8,0.09386879205703735
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,5120,fp8,0.09053546190261841
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,4096,fp8,0.08395093282063802
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,3584,fp8,0.08049173355102537
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,3072,fp8,0.07835413018862407
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,2560,fp8,0.07435946464538574
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,2048,fp8,0.07315413355827331
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,1536,fp8,0.06782933473587036
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,1024,fp8,0.06733973423639933
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,768,fp8,0.06595306595166525
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,512,fp8,0.06392853260040283
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,256,fp8,0.06295466820398968
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,128,fp8,0.06197333335876465
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,64,fp8,0.06145919760068258
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,32,fp8,0.06212693552176157
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,192,65536,fp8,0.14487999280293784
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,16384,fp8,0.10458665688832602
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,12288,fp8,0.09908586740493774
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,10240,fp8,0.09675520261128742
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,8192,fp8,0.09302614132563274
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,7168,fp8,0.09329813718795775
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,6144,fp8,0.09209386905034384
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,5120,fp8,0.08577173550923664
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,4096,fp8,0.07963733673095703
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,3584,fp8,0.07574293216069539
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,3072,fp8,0.07497813304265341
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,2560,fp8,0.07250026464462281
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,2048,fp8,0.0687818686167399
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,1536,fp8,0.06831573049227396
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,1024,fp8,0.06731626590092976
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,768,fp8,0.06451519727706909
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,512,fp8,0.06403839886188507
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,256,fp8,0.06253332992394765
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,128,fp8,0.06143999894460042
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,64,fp8,0.06293333073457082
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,32,fp8,0.06183466911315918
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,160,65536,fp8,0.14543573061625162
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,16384,fp8,0.10629547039667764
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,12288,fp8,0.09723306496938068
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,10240,fp8,0.09458239873250326
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,8192,fp8,0.09222720464070638
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,7168,fp8,0.0871882677078247
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,6144,fp8,0.0872543970743815
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,5120,fp8,0.08033600052197774
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,4096,fp8,0.07433600227038066
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,3584,fp8,0.07286719878514608
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,3072,fp8,0.07221013307571411
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,2560,fp8,0.06838506857554118
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,2048,fp8,0.06904746890068053
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,1536,fp8,0.06727679967880248
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,1024,fp8,0.06497493187586466
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,768,fp8,0.06469119787216186
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,512,fp8,0.06342720091342927
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,256,fp8,0.06251093447208406
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,128,fp8,0.06279467046260832
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,64,fp8,0.061389867464701334
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,32,fp8,0.06173759798208873
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,128,65536,fp8,0.12328319549560549
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,16384,fp8,0.09424320062001545
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,12288,fp8,0.09334294001261392
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,10240,fp8,0.09036586284637452
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,8192,fp8,0.08457813660303752
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,7168,fp8,0.080458668867747
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,6144,fp8,0.07588693499565125
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,5120,fp8,0.07452372908592222
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,4096,fp8,0.07224533160527546
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,3584,fp8,0.07179093559583027
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,3072,fp8,0.06687039732933045
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,2560,fp8,0.06764586369196574
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,2048,fp8,0.0677013337612152
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,1536,fp8,0.06465600331624348
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,1024,fp8,0.06468480428059895
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,768,fp8,0.0635061373313268
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,512,fp8,0.062377602855364485
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,256,fp8,0.06306346952915191
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,128,fp8,0.061356800794601436
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,64,fp8,0.062181334694226584
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,32,fp8,0.06144533554712931
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,96,65536,fp8,0.12224853833516439
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,16384,fp8,0.09626560211181641
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,12288,fp8,0.08923306862513224
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,10240,fp8,0.08321493466695148
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,8192,fp8,0.07932266990343728
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,7168,fp8,0.07645546197891236
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,6144,fp8,0.07480640212694804
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,5120,fp8,0.07313920458157858
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,4096,fp8,0.0676159958044688
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,3584,fp8,0.06851200262705485
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,3072,fp8,0.06861866315205892
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,2560,fp8,0.06682879726092023
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,2048,fp8,0.06691946387290955
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,1536,fp8,0.06447573105494181
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,1024,fp8,0.06376852989196778
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,768,fp8,0.06354986230532328
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,512,fp8,0.06328853170077006
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,256,fp8,0.06290559868017834
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,128,fp8,0.0614869366089503
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,64,fp8,0.060711464285850524
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,32,fp8,0.061530663569768274
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,80,65536,fp8,0.11411945819854735
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,16384,fp8,0.09336853027343749
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,12288,fp8,0.08304959932963052
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,10240,fp8,0.07938666741053264
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,8192,fp8,0.07449920376141866
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,7168,fp8,0.07400746941566466
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,6144,fp8,0.07149440248807272
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,5120,fp8,0.068093870083491
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,4096,fp8,0.06795093218485514
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,3584,fp8,0.0666645328203837
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,3072,fp8,0.06735146443049113
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,2560,fp8,0.06757333676020305
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,2048,fp8,0.06497173110644022
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,1536,fp8,0.06410346428553262
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,1024,fp8,0.06354453066984812
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,768,fp8,0.06258559723695119
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,512,fp8,0.06261759897073109
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,256,fp8,0.061550932129224134
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,128,fp8,0.06121173004309337
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,64,fp8,0.06190293629964192
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,32,fp8,0.06100373367468516
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,64,65536,fp8,0.10263253053029378
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,16384,fp8,0.08488426605860393
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,12288,fp8,0.07500586907068889
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,10240,fp8,0.07615999778111776
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,8192,fp8,0.07298773129781087
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,7168,fp8,0.06956266363461812
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,6144,fp8,0.06854613622029622
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,5120,fp8,0.0673077364762624
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,4096,fp8,0.06742293039957682
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,3584,fp8,0.0672661324342092
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,3072,fp8,0.06502613027890523
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,2560,fp8,0.06443093617757162
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,2048,fp8,0.06501973470052083
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,1536,fp8,0.06414613127708435
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,1024,fp8,0.06284906466801962
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,768,fp8,0.06241066753864289
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,512,fp8,0.062156798442204786
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,256,fp8,0.0617162674665451
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,128,fp8,0.06207679907480875
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,64,fp8,0.06098666489124298
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,32,fp8,0.0614250659942627
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,48,65536,fp8,0.1098965326944987
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,16384,fp8,0.07517759998639426
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,12288,fp8,0.07142719626426697
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,10240,fp8,0.06949440042177836
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,8192,fp8,0.06786560416221618
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,7168,fp8,0.06771093606948853
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,6144,fp8,0.06761280099550884
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,5120,fp8,0.06684373021125793
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,4096,fp8,0.0645098646481832
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,3584,fp8,0.06483093698819478
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,3072,fp8,0.06380800406138101
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,2560,fp8,0.06398719648520151
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,2048,fp8,0.06352319916089375
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,1536,fp8,0.06244586904843648
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,1024,fp8,0.06250880360603334
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,768,fp8,0.0618890643119812
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,512,fp8,0.06162133018175761
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,256,fp8,0.062360533078511554
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,128,fp8,0.060854397217432646
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,64,fp8,0.061564803123474114
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,32,fp8,0.061571201682090765
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,32,65536,fp8,0.1003893295923869
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,16384,fp8,0.06758400003115336
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,12288,fp8,0.0679253339767456
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,10240,fp8,0.06710506876309713
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,8192,fp8,0.06503253181775412
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,7168,fp8,0.06490559776624043
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,6144,fp8,0.06363839904467265
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,5120,fp8,0.06373119950294495
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,4096,fp8,0.06365013718605042
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,3584,fp8,0.06249386668205262
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,3072,fp8,0.06360426644484202
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,2560,fp8,0.06245120267073313
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,2048,fp8,0.06296000281969706
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,1536,fp8,0.06319999794165294
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,1024,fp8,0.06150080064932505
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,768,fp8,0.06141333281993867
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,512,fp8,0.062441597382227584
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,256,fp8,0.06052266756693522
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,128,fp8,0.06156586607297262
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,64,fp8,0.06150720020135244
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,32,fp8,0.060964266459147126
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,16,65536,fp8,0.10011946360270184
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,16384,fp8,0.0647871990998586
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,12288,fp8,0.0641482651233673
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,10240,fp8,0.06372693280378977
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,8192,fp8,0.06346986691157022
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,7168,fp8,0.06386559704939523
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,6144,fp8,0.06334613064924877
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,5120,fp8,0.06230826576550802
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,4096,fp8,0.062345598141352335
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,3584,fp8,0.062251735726992295
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,3072,fp8,0.06171840131282807
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,2560,fp8,0.06263040204842886
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,2048,fp8,0.0628853311141332
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,1536,fp8,0.06160532931486765
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,1024,fp8,0.061332264542579645
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,768,fp8,0.06221653521060943
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,512,fp8,0.06040533185005188
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,256,fp8,0.06149226427078247
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,128,fp8,0.061599996685981755
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,64,fp8,0.06074773669242858
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,32,fp8,0.06122986773649851
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,8,65536,fp8,0.0746730665365855
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,16384,fp8,0.06347093582153321
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,12288,fp8,0.06346026857693991
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,10240,fp8,0.06267626980940501
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,8192,fp8,0.06260373691717783
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,7168,fp8,0.06252906719843546
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,6144,fp8,0.06201813419659933
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,5120,fp8,0.06241386930147807
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,4096,fp8,0.06199679772059122
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,3584,fp8,0.06122559905052184
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,3072,fp8,0.06271679699420929
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,2560,fp8,0.06134399672349294
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,2048,fp8,0.06228693425655365
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,1536,fp8,0.06221226652463277
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,1024,fp8,0.06065919895966848
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,768,fp8,0.06163413524627686
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,512,fp8,0.06089173555374145
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,256,fp8,0.0609194686015447
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,128,fp8,0.06148159901301066
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,64,fp8,0.06117866535981496
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,32,fp8,0.06162240107854208
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,4,65536,fp8,0.06773866812388102
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,16384,fp8,0.062438402573267625
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,12288,fp8,0.06171840230623881
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,10240,fp8,0.06266560157140096
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,8192,fp8,0.06320533553759257
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,7168,fp8,0.06141759653886159
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,6144,fp8,0.06226026316483815
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,5120,fp8,0.06133333047231039
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,4096,fp8,0.06098239719867707
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,3584,fp8,0.06218347052733103
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,3072,fp8,0.061019731561342876
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,2560,fp8,0.061815470457077026
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,2048,fp8,0.061289599537849425
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,1536,fp8,0.06115413010120392
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,1024,fp8,0.061600002646446224
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,768,fp8,0.06087999641895294
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,512,fp8,0.06096426645914714
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,256,fp8,0.06141653259595236
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,128,fp8,0.060898129145304354
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,64,fp8,0.06160853703816732
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,32,fp8,0.06154879927635192
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,2,65536,fp8,0.06455892920494079
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,16384,fp8,0.06296639641125998
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,12288,fp8,0.06147200365861257
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,10240,fp8,0.06153279940287272
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,8192,fp8,0.06253333687782286
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,7168,fp8,0.061230931679407755
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,6144,fp8,0.062042664488156644
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,5120,fp8,0.06184640328089396
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,4096,fp8,0.0605973333120346
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,3584,fp8,0.06159040133158366
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,3072,fp8,0.060976001620292666
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,2560,fp8,0.06100159982840221
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,2048,fp8,0.06156693001588186
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,1536,fp8,0.06118186314900716
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,1024,fp8,0.06096533238887788
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,768,fp8,0.06105813284715017
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,512,fp8,0.061300264795621236
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,256,fp8,0.06113599936167399
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,128,fp8,0.06181973417599996
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,64,fp8,0.06177493333816528
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,32,fp8,0.061426135897636416
TRTLLM,1.0.0rc6,NVIDIA B200,compute_scale,torch_ops,1,65536,fp8,0.06373226344585418
