framework,version,device,op_name,kernel_source,m,k,quant_dtype,latency
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,16384,fp8,0.20304320653279617
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,12288,fp8,0.15447147687276208
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,10240,fp8,0.1303552150726318
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,8192,fp8,0.10564800898234047
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,7168,fp8,0.09325012365976972
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,6144,fp8,0.07988907496134438
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,5120,fp8,0.0668234666188558
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,4096,fp8,0.0536970615386963
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,3584,fp8,0.04670399824778239
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,3072,fp8,0.03927893241246542
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,2560,fp8,0.030638929208119704
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,2048,fp8,0.02361279924710591
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,1536,fp8,0.020498134692509967
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,1024,fp8,0.012670934200286869
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,768,fp8,0.012156800429026286
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,512,fp8,0.009714132547378539
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,256,fp8,0.008552532891432443
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,128,fp8,0.008596266309420267
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,64,fp8,0.007486933221419653
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,32,fp8,0.007154132922490438
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8192,65536,fp8,0.7842037836710611
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,16384,fp8,0.10592319170633951
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,12288,fp8,0.08065172831217447
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,10240,fp8,0.0671882629394531
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,8192,fp8,0.05406186580657958
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,7168,fp8,0.04753066698710125
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,6144,fp8,0.03977493047714233
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,5120,fp8,0.030115195115407313
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,4096,fp8,0.023537065585454306
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,3584,fp8,0.02413120071093241
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,3072,fp8,0.020531201362609858
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,2560,fp8,0.015048533678054806
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,2048,fp8,0.012577066818873089
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,1536,fp8,0.012193066875139873
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,1024,fp8,0.009624533355236053
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,768,fp8,0.009111467003822328
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,512,fp8,0.008534399668375652
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,256,fp8,0.007912533233563106
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,128,fp8,0.007277866701285044
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,64,fp8,0.007105066627264023
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,32,fp8,0.006802132974068323
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4096,65536,fp8,0.39838399887084963
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,16384,fp8,0.05401600201924643
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,12288,fp8,0.03944746653238932
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,10240,fp8,0.03009386857350667
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,8192,fp8,0.02289599974950155
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,7168,fp8,0.0252778689066569
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,6144,fp8,0.020403202374776202
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,5120,fp8,0.01608533263206482
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,4096,fp8,0.012451199690500894
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,3584,fp8,0.012115200360616047
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,3072,fp8,0.010991999506950378
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,2560,fp8,0.010411733388900758
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,2048,fp8,0.009955200056234995
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,1536,fp8,0.009258667131265004
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,1024,fp8,0.008061866462230683
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,768,fp8,0.007964800049861272
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,512,fp8,0.008642133076985676
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,256,fp8,0.007279999554157256
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,128,fp8,0.006907733281453449
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,64,fp8,0.006588800003131232
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,32,fp8,0.0069322663048903155
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2048,65536,fp8,0.20317120552062984
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,16384,fp8,0.024140797058741253
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,12288,fp8,0.020947202046712243
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,10240,fp8,0.015047467748324078
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,8192,fp8,0.01267199913660685
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,7168,fp8,0.011530667543411255
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,6144,fp8,0.011387733618418374
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,5120,fp8,0.010686933000882467
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,4096,fp8,0.009561599791049959
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,3584,fp8,0.009300266206264498
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,3072,fp8,0.00941226730744044
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,2560,fp8,0.00850666711727778
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,2048,fp8,0.008508800466855367
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,1536,fp8,0.007868799567222596
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,1024,fp8,0.00769493281841278
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,768,fp8,0.007293866574764253
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,512,fp8,0.007072000453869502
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,256,fp8,0.007192533711592357
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,128,fp8,0.006776533275842666
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,64,fp8,0.006867200136184692
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,32,fp8,0.006697600334882736
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1024,65536,fp8,0.10592532952626546
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,16384,fp8,0.020577065149943032
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,12288,fp8,0.014725332458813984
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,10240,fp8,0.012465065717697146
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,8192,fp8,0.011231999595959979
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,7168,fp8,0.010539732376734416
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,6144,fp8,0.009985066950321198
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,5120,fp8,0.009308800101280209
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,4096,fp8,0.008828799923261006
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,3584,fp8,0.009363199770450591
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,3072,fp8,0.008473599950472515
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,2560,fp8,0.008118399977684022
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,2048,fp8,0.007620266576608022
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,1536,fp8,0.007890133062998454
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,1024,fp8,0.0073578663170337675
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,768,fp8,0.007287467022736867
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,512,fp8,0.00728426699837049
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,256,fp8,0.007092266529798508
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,128,fp8,0.007167999694744745
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,64,fp8,0.006827733168999354
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,32,fp8,0.007040000458558401
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,768,65536,fp8,0.08106239636739095
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,16384,fp8,0.013086932897567745
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,12288,fp8,0.011384533842404681
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,10240,fp8,0.010801066954930624
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,8192,fp8,0.009479466577370959
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,7168,fp8,0.009659733871618907
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,6144,fp8,0.00893013377984365
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,5120,fp8,0.009022933741410574
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,4096,fp8,0.008357333143552144
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,3584,fp8,0.008677333096663157
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,3072,fp8,0.007925333579381307
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,2560,fp8,0.007966933151086172
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,2048,fp8,0.00826666628321012
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,1536,fp8,0.007333333541949589
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,1024,fp8,0.007606399556001028
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,768,fp8,0.007146666944026946
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,512,fp8,0.006940799951553344
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,256,fp8,0.007342933118343353
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,128,fp8,0.006982400268316269
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,64,fp8,0.006920533130566279
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,32,fp8,0.006945066402355829
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,512,65536,fp8,0.05396053791046142
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,16384,fp8,0.011516800522804262
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,12288,fp8,0.010237866640090944
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,10240,fp8,0.009879466394583383
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,8192,fp8,0.008883200089136758
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,7168,fp8,0.009324800471464792
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,6144,fp8,0.008523733417193095
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,5120,fp8,0.008104532957077026
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,4096,fp8,0.008213333785533905
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,3584,fp8,0.007762133330106735
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,3072,fp8,0.007502933591604232
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,2560,fp8,0.007796267171700795
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,2048,fp8,0.0074389331042766585
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,1536,fp8,0.007236266136169434
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,1024,fp8,0.007231999685366948
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,768,fp8,0.007261867076158524
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,512,fp8,0.006782933572928111
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,256,fp8,0.007037866363922755
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,128,fp8,0.006684799740711848
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,64,fp8,0.0067658662796020504
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,32,fp8,0.006611199925343196
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,384,65536,fp8,0.03932373126347861
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,16384,fp8,0.009361066917578379
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,12288,fp8,0.00902186632156372
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,10240,fp8,0.008844800293445587
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,8192,fp8,0.008044800162315367
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,7168,fp8,0.00810773322979609
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,6144,fp8,0.007815466572841007
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,5120,fp8,0.008571733285983403
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,4096,fp8,0.0077866663535436
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,3584,fp8,0.007300266375144322
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,3072,fp8,0.007286400347948073
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,2560,fp8,0.007282132903734844
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,2048,fp8,0.00725973347822825
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,1536,fp8,0.007187200337648391
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,1024,fp8,0.0070848003029823315
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,768,fp8,0.007307732850313186
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,512,fp8,0.006991999596357346
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,256,fp8,0.007137067119280497
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,128,fp8,0.006904533753792444
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,64,fp8,0.006730667004982629
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,32,fp8,0.006855466713507971
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,256,65536,fp8,0.023100801308949793
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,16384,fp8,0.00896639972925186
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,12288,fp8,0.008663467069466909
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,10240,fp8,0.008224000036716462
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,8192,fp8,0.008139733721812568
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,7168,fp8,0.008833066125710803
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,6144,fp8,0.0077610664069652545
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,5120,fp8,0.007923199484745663
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,4096,fp8,0.007278933624426523
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,3584,fp8,0.007732266187667847
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,3072,fp8,0.0075775995850563055
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,2560,fp8,0.007238400479157766
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,2048,fp8,0.007145599772532781
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,1536,fp8,0.00724480003118515
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,1024,fp8,0.007099733750025431
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,768,fp8,0.006877866884072622
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,512,fp8,0.007426133255163829
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,256,fp8,0.007054933408896128
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,128,fp8,0.006720000008742014
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,64,fp8,0.007012266914049785
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,32,fp8,0.006645333518584568
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,192,65536,fp8,0.02041813135147095
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,16384,fp8,0.008686932921409606
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,12288,fp8,0.008348800241947174
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,10240,fp8,0.008307199676831564
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,8192,fp8,0.007958400001128515
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,7168,fp8,0.007988266398509344
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,6144,fp8,0.007700266689062119
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,5120,fp8,0.00722880040605863
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,4096,fp8,0.007200000186761219
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,3584,fp8,0.0075082669655481965
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,3072,fp8,0.0072981332739194245
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,2560,fp8,0.0074357338249683375
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,2048,fp8,0.007370666166146596
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,1536,fp8,0.007064532985289891
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,1024,fp8,0.007109332829713821
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,768,fp8,0.007114666452010473
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,512,fp8,0.006940799951553345
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,256,fp8,0.007061333705981573
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,128,fp8,0.006541866809129714
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,64,fp8,0.006974933544794717
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,32,fp8,0.006737066805362701
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,160,65536,fp8,0.014705066879590353
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,16384,fp8,0.00830293297767639
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,12288,fp8,0.007942400376001994
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,10240,fp8,0.007786666601896287
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,8192,fp8,0.008141866326332093
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,7168,fp8,0.007948799679676693
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,6144,fp8,0.007327999671300252
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,5120,fp8,0.0074175996085008
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,4096,fp8,0.007571200033028922
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,3584,fp8,0.00730773334701856
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,3072,fp8,0.00733440046509107
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,2560,fp8,0.007388799637556076
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,2048,fp8,0.006999466816584269
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,1536,fp8,0.007156266272068023
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,1024,fp8,0.006951466947793961
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,768,fp8,0.007229866584142049
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,512,fp8,0.0067733330031236015
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,256,fp8,0.006853333860635758
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,128,fp8,0.006589866429567337
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,64,fp8,0.006503467013438543
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,32,fp8,0.006593066453933715
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,128,65536,fp8,0.01259093284606934
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,16384,fp8,0.008016000439723333
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,12288,fp8,0.0074997335672378545
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,10240,fp8,0.00754346673687299
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,8192,fp8,0.00734720006585121
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,7168,fp8,0.007085866977771123
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,6144,fp8,0.007422933230797449
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,5120,fp8,0.007091199606657028
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,4096,fp8,0.007091199855009714
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,3584,fp8,0.007023999591668446
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,3072,fp8,0.007079466929038365
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,2560,fp8,0.007271466652552287
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,2048,fp8,0.006855466961860656
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,1536,fp8,0.006849066664775213
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,1024,fp8,0.007087999830643336
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,768,fp8,0.0069141333301862085
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,512,fp8,0.006753066927194595
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,256,fp8,0.006571733703215916
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,128,fp8,0.007368532816569011
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,64,fp8,0.006707199911276499
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,32,fp8,0.006587733825047811
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,96,65536,fp8,0.0109418660402298
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,16384,fp8,0.007993600517511367
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,12288,fp8,0.007372800509134929
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,10240,fp8,0.007371733337640763
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,8192,fp8,0.007108266403277715
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,7168,fp8,0.007398400207360585
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,6144,fp8,0.007048533111810685
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,5120,fp8,0.007142400244871775
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,4096,fp8,0.007097600152095158
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,3584,fp8,0.006995199869076411
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,3072,fp8,0.007182933141787846
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,2560,fp8,0.006808533022801081
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,2048,fp8,0.007117866476376852
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,1536,fp8,0.007065600156784058
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,1024,fp8,0.00681066612402598
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,768,fp8,0.007347200314203899
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,512,fp8,0.0066592000424861904
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,256,fp8,0.006489600241184235
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,128,fp8,0.006876800457636516
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,64,fp8,0.00656213363011678
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,32,fp8,0.006718933085600535
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,80,65536,fp8,0.0102047989765803
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,16384,fp8,0.007590399930874505
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,12288,fp8,0.007060266782840094
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,10240,fp8,0.007170133541027705
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,8192,fp8,0.00746026709675789
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,7168,fp8,0.007225600381692251
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,6144,fp8,0.007601066182057064
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,5120,fp8,0.007539199789365132
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,4096,fp8,0.00714026689529419
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,3584,fp8,0.007151999572912852
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,3072,fp8,0.006908800452947617
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,2560,fp8,0.0070826664566993715
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,2048,fp8,0.007065599660078685
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,1536,fp8,0.006825599819421768
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,1024,fp8,0.007125333199898401
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,768,fp8,0.006638932724793751
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,512,fp8,0.006987733642260234
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,256,fp8,0.0068618667622407275
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,128,fp8,0.006839466591676076
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,64,fp8,0.006216532985369364
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,32,fp8,0.006568533182144164
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,64,65536,fp8,0.009902934233347574
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,16384,fp8,0.007203200211127599
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,12288,fp8,0.007714133212963741
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,10240,fp8,0.007622399926185608
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,8192,fp8,0.007202133039633433
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,7168,fp8,0.007268267124891282
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,6144,fp8,0.0070570670068264
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,5120,fp8,0.007269333551327387
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,4096,fp8,0.007102933277686438
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,3584,fp8,0.006877866635719935
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,3072,fp8,0.007395199934641519
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,2560,fp8,0.006979199747244518
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,2048,fp8,0.007276800026496251
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,1536,fp8,0.006983466694752376
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,1024,fp8,0.006874666363000869
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,768,fp8,0.007019732892513275
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,512,fp8,0.006821333120266597
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,256,fp8,0.006915200253327688
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,128,fp8,0.006626133620738982
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,64,fp8,0.006696532915035884
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,32,fp8,0.006910933554172516
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,48,65536,fp8,0.008997333546479542
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,16384,fp8,0.007597866654396057
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,12288,fp8,0.0072746669252713525
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,10240,fp8,0.007331200192372005
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,8192,fp8,0.00694506714741389
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,7168,fp8,0.007026133686304093
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,6144,fp8,0.00711253359913826
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,5120,fp8,0.0071776005129019415
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,4096,fp8,0.0069141328334808355
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,3584,fp8,0.007260799904664357
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,3072,fp8,0.007175466914971669
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,2560,fp8,0.007051733632882437
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,2048,fp8,0.007073066135247549
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,1536,fp8,0.006761599580446879
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,1024,fp8,0.0069514664510885865
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,768,fp8,0.006947199503580729
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,512,fp8,0.009948799759149552
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,256,fp8,0.006891733656326929
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,128,fp8,0.006720000008742014
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,64,fp8,0.006392533580462137
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,32,fp8,0.00699093316992124
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,32,65536,fp8,0.008248532811800637
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,16384,fp8,0.007220266511042912
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,12288,fp8,0.0071552000939846035
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,10240,fp8,0.007195733487606049
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,8192,fp8,0.006947200000286102
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,7168,fp8,0.0071221329271793365
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,6144,fp8,0.007236266632874807
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,5120,fp8,0.007132800420125325
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,4096,fp8,0.007174399991830191
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,3584,fp8,0.007437866926193237
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,3072,fp8,0.006631466497977575
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,2560,fp8,0.0070453333357969935
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,2048,fp8,0.00699733297030131
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,1536,fp8,0.006785066425800325
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,1024,fp8,0.007099733750025431
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,768,fp8,0.006734933704137801
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,512,fp8,0.006941866874694825
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,256,fp8,0.006444800148407618
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,128,fp8,0.006783999999364219
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,64,fp8,0.00666026622056961
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,32,fp8,0.006786133100589116
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,16,65536,fp8,0.00802133356531461
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,16384,fp8,0.007058133433262507
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,12288,fp8,0.007077333082755407
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,10240,fp8,0.006746666878461837
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,8192,fp8,0.006806399921576183
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,7168,fp8,0.006920533130566279
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,6144,fp8,0.006588800251483918
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,5120,fp8,0.006965332974990209
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,4096,fp8,0.006837333490451177
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,3584,fp8,0.00664426659544309
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,3072,fp8,0.006795732925335566
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,2560,fp8,0.006596266726652781
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,2048,fp8,0.006727466980616252
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,1536,fp8,0.0067274667322635645
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,1024,fp8,0.006722133606672287
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,768,fp8,0.006533333659172058
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,512,fp8,0.006438399851322174
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,256,fp8,0.0066730670630931865
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,128,fp8,0.006608000397682189
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,64,fp8,0.006805333495140076
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,32,fp8,0.006808533271153768
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,8,65536,fp8,0.0074346664051214845
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,16384,fp8,0.00689706653356552
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,12288,fp8,0.006649599721034367
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,10240,fp8,0.006939733525117239
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,8192,fp8,0.006825599819421768
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,7168,fp8,0.006625067194302878
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,6144,fp8,0.006842667112747828
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,5120,fp8,0.006780799726645153
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,4096,fp8,0.006617599725723267
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,3584,fp8,0.007076266407966614
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,3072,fp8,0.006798933446407317
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,2560,fp8,0.006680533041556676
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,2048,fp8,0.00685546671350797
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,1536,fp8,0.00667946661512057
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,1024,fp8,0.006737066805362702
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,768,fp8,0.006937600175539653
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,512,fp8,0.006563200553258261
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,256,fp8,0.006972800195217132
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,128,fp8,0.006700800110896427
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,64,fp8,0.006727466483910878
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,32,fp8,0.006947199751933415
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,4,65536,fp8,0.00687466710805893
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,16384,fp8,0.007026133189598719
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,12288,fp8,0.006933333476384481
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,10240,fp8,0.0068437327941258735
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,8192,fp8,0.0066197330753008515
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,7168,fp8,0.007011200239260991
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,6144,fp8,0.006910932809114456
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,5120,fp8,0.0066207995017369585
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,4096,fp8,0.006902400404214859
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,3584,fp8,0.006661332895358404
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,3072,fp8,0.006601599603891373
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,2560,fp8,0.006822400043408076
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,2048,fp8,0.006543999910354614
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,1536,fp8,0.006993066271146138
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,1024,fp8,0.006758400052785873
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,768,fp8,0.00670613373319308
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,512,fp8,0.006971733023722966
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,256,fp8,0.006808533271153768
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,128,fp8,0.0067893336216608684
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,64,fp8,0.006589867174625397
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,32,fp8,0.006801067044337591
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,2,65536,fp8,0.00680426706870397
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,16384,fp8,0.0065407996376355484
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,12288,fp8,0.006961067020893097
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,10240,fp8,0.006563200056552887
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,8192,fp8,0.006746666381756463
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,7168,fp8,0.006721066186825435
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,6144,fp8,0.006532266487677891
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,5120,fp8,0.006711466858784357
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,4096,fp8,0.006392533332109452
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,3584,fp8,0.006626133372386297
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,3072,fp8,0.0068490669131279
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,2560,fp8,0.006502400090297063
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,2048,fp8,0.006681600213050842
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,1536,fp8,0.006555733581384023
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,1024,fp8,0.006827732920646668
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,768,fp8,0.00655573308467865
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,512,fp8,0.0068181338409582775
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,256,fp8,0.006868266314268113
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,128,fp8,0.006583466877539953
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,64,fp8,0.006794666747252147
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,32,fp8,0.0065610667069753
TRTLLM,1.0.0rc3,NVIDIA H200,compute_scale,torch_ops,1,65536,fp8,0.006853333612283069
