framework,version,device,op_name,kernel_source,m,k,quant_dtype,latency
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,16384,fp8,0.21392745971679689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,12288,fp8,0.1623082637786865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,10240,fp8,0.13747305870056148
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,8192,fp8,0.11117013295491535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,7168,fp8,0.09853546619415283
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,6144,fp8,0.08506346543629963
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,5120,fp8,0.0711135943730672
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,4096,fp8,0.05729386806488036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,3584,fp8,0.050449069341023764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,3072,fp8,0.04259413480758668
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,2560,fp8,0.03474453290303548
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,2048,fp8,0.024459735552469893
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,1536,fp8,0.023087998231252037
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,1024,fp8,0.012882134318351746
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,768,fp8,0.011149867375691732
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,512,fp8,0.009656533102194467
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,256,fp8,0.008440532783667246
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,128,fp8,0.007953066130479176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,64,fp8,0.007559466858704885
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,32,fp8,0.007176532844702403
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8192,65536,fp8,0.8333269755045576
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,16384,fp8,0.11122986475626628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,12288,fp8,0.08522026538848877
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,10240,fp8,0.07182079950968424
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,8192,fp8,0.057476266225179054
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,7168,fp8,0.05064639647801718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,6144,fp8,0.04330560366312662
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,5120,fp8,0.03514240185419718
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,4096,fp8,0.025138131777445477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,3584,fp8,0.022877864042917886
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,3072,fp8,0.02244480053583781
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,2560,fp8,0.017797333002090448
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,2048,fp8,0.012957865993181866
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,1536,fp8,0.011238401134808857
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,1024,fp8,0.009816533823808034
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,768,fp8,0.009177599350611368
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,512,fp8,0.008243199686209363
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,256,fp8,0.008639999727408091
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,128,fp8,0.007642666250467299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,64,fp8,0.007226667056481042
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,32,fp8,0.0071274663011233015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4096,65536,fp8,0.4203338623046875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,16384,fp8,0.057692805926005036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,12288,fp8,0.042686935265858955
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,10240,fp8,0.034494928518931066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,8192,fp8,0.02424533367156982
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,7168,fp8,0.021287467082341513
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,6144,fp8,0.022957865397135423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,5120,fp8,0.018223999937375387
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,4096,fp8,0.012513067324956257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,3584,fp8,0.011995733777681986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,3072,fp8,0.010948266585667929
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,2560,fp8,0.010570667187372844
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,2048,fp8,0.009738667309284209
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,1536,fp8,0.00914453367392222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,1024,fp8,0.00825279951095581
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,768,fp8,0.007827199747165045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,512,fp8,0.008089600006739299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,256,fp8,0.007116799801588058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,128,fp8,0.0074357333282629645
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,64,fp8,0.006883199761311213
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,32,fp8,0.00698773389061292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2048,65536,fp8,0.21469335556030278
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,16384,fp8,0.024960001309712723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,12288,fp8,0.02262719869613648
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,10240,fp8,0.017815465728441875
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,8192,fp8,0.013260799646377565
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,7168,fp8,0.011527467767397563
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,6144,fp8,0.011144532759984335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,5120,fp8,0.010250665744145711
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,4096,fp8,0.009708800415198008
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,3584,fp8,0.009740799665451052
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,3072,fp8,0.008727466066678364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,2560,fp8,0.00936533361673355
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,2048,fp8,0.008353066444396973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,1536,fp8,0.007962667196989058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,1024,fp8,0.008238933483759564
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,768,fp8,0.007709866513808569
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,512,fp8,0.007342933615048727
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,256,fp8,0.007140266646941502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,128,fp8,0.006876799712578456
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,64,fp8,0.007107199976841607
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,32,fp8,0.00677653377254804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1024,65536,fp8,0.11151466369628904
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,16384,fp8,0.02261546651522318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,12288,fp8,0.015622399250666299
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,10240,fp8,0.012741332252820332
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,8192,fp8,0.011185067892074586
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,7168,fp8,0.010680532455444335
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,6144,fp8,0.010261332492033643
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,5120,fp8,0.010734933614730834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,4096,fp8,0.009538133442401887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,3584,fp8,0.009358933568000794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,3072,fp8,0.008563199639320375
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,2560,fp8,0.00845226695140203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,2048,fp8,0.007983999947706858
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,1536,fp8,0.008150399724642435
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,1024,fp8,0.007256532957156499
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,768,fp8,0.007676799595355988
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,512,fp8,0.007369600236415864
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,256,fp8,0.0075317338109016415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,128,fp8,0.00695466622710228
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,64,fp8,0.007100800176461537
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,32,fp8,0.0068138663967450466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,768,65536,fp8,0.08520534038543702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,16384,fp8,0.013125333189964297
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,12288,fp8,0.011951998869578043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,10240,fp8,0.010555734237035116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,8192,fp8,0.009990401069323222
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,7168,fp8,0.009395200014114381
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,6144,fp8,0.009019733468691508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,5120,fp8,0.008942933380603792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,4096,fp8,0.008237866560618081
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,3584,fp8,0.008871466914812724
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,3072,fp8,0.007783466577529907
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,2560,fp8,0.007993600020805994
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,2048,fp8,0.007557333260774611
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,1536,fp8,0.007708800335725149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,1024,fp8,0.007301333049933116
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,768,fp8,0.007356799642244974
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,512,fp8,0.007530666639407477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,256,fp8,0.0070208000640074415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,128,fp8,0.0072586668034394595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,64,fp8,0.006964266548554103
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,32,fp8,0.007021866738796233
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,512,65536,fp8,0.058092800776163744
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,16384,fp8,0.011341866850852965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,12288,fp8,0.009902933736642201
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,10240,fp8,0.009257599711418155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,8192,fp8,0.009537066022555031
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,7168,fp8,0.008648533622423809
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,6144,fp8,0.008677333096663157
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,5120,fp8,0.008161066472530365
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,4096,fp8,0.00816639984647433
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,3584,fp8,0.007958400497833888
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,3072,fp8,0.007809066772460938
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,2560,fp8,0.007603200525045395
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,2048,fp8,0.007374933610359828
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,1536,fp8,0.0077738665044307705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,1024,fp8,0.007251199583212535
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,768,fp8,0.007883733759323755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,512,fp8,0.007185066739718119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,256,fp8,0.007185066739718119
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,128,fp8,0.006952532877524694
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,64,fp8,0.0069962667922178895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,32,fp8,0.00675626719991366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,384,65536,fp8,0.04282133579254151
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,16384,fp8,0.009677867094675698
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,12288,fp8,0.009251200159390769
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,10240,fp8,0.008487466474374135
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,8192,fp8,0.008134399851163227
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,7168,fp8,0.007864532868067422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,6144,fp8,0.008412799735864002
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,5120,fp8,0.008172799895207087
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,4096,fp8,0.00763626645008723
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,3584,fp8,0.007981866598129272
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,3072,fp8,0.007505066692829132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,2560,fp8,0.007831466694672903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,2048,fp8,0.00749440019329389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,1536,fp8,0.007410133878389995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,1024,fp8,0.0073514665166536965
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,768,fp8,0.0070517333845297485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,512,fp8,0.006957866748174032
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,256,fp8,0.0072554665307203925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,128,fp8,0.006939733773469925
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,64,fp8,0.00703253298997879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,32,fp8,0.006733866284290949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,256,65536,fp8,0.024661334355672207
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,16384,fp8,0.009255466361840567
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,12288,fp8,0.008398933708667755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,10240,fp8,0.008489599823951722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,8192,fp8,0.008941866954167686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,7168,fp8,0.008007466793060303
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,6144,fp8,0.007672533392906189
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,5120,fp8,0.007829332848389943
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,4096,fp8,0.007603199779987337
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,3584,fp8,0.00762880047162374
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,3072,fp8,0.0072714666525522855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,2560,fp8,0.007544533163309098
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,2048,fp8,0.0071327996750672655
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,1536,fp8,0.007272533824046453
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,1024,fp8,0.006951466947793961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,768,fp8,0.007080533107121786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,512,fp8,0.0069930667678515124
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,256,fp8,0.00684799998998642
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,128,fp8,0.006872533510128658
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,64,fp8,0.006781866898139318
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,32,fp8,0.006958932926257451
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,192,65536,fp8,0.02339306672414144
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,16384,fp8,0.008762667079766591
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,12288,fp8,0.008141866823037466
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,10240,fp8,0.00818879952033361
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,8192,fp8,0.008849066992600761
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,7168,fp8,0.0077173329889774316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,6144,fp8,0.007577600330114364
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,5120,fp8,0.007417599856853485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,4096,fp8,0.007396267106135686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,3584,fp8,0.007330133020877837
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,3072,fp8,0.007445332904656728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,2560,fp8,0.007268266379833221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,2048,fp8,0.006986666719118754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,1536,fp8,0.007200000186761221
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,1024,fp8,0.007017600039641062
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,768,fp8,0.007281066725651423
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,512,fp8,0.006915199756622314
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,256,fp8,0.007089067002137501
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,128,fp8,0.006876800457636515
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,64,fp8,0.006911999732255935
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,32,fp8,0.006683733810981114
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,160,65536,fp8,0.018661333123842876
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,16384,fp8,0.008593067030111949
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,12288,fp8,0.008092799534400305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,10240,fp8,0.0074762667218844095
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,8192,fp8,0.007618133227030437
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,7168,fp8,0.00740479975938797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,6144,fp8,0.007758933554093043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,5120,fp8,0.007418667028347651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,4096,fp8,0.007452800124883651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,3584,fp8,0.007487999896208445
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,3072,fp8,0.007435733576615651
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,2560,fp8,0.007309866944948833
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,2048,fp8,0.007249066730340323
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,1536,fp8,0.007264000425736109
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,1024,fp8,0.006975999722878138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,768,fp8,0.007176533589760462
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,512,fp8,0.006986666222413382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,256,fp8,0.007169066121180852
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,128,fp8,0.006757333626349767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,64,fp8,0.0070314668118953705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,32,fp8,0.006825599819421767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,128,65536,fp8,0.01319573422273
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,16384,fp8,0.008083199958006541
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,12288,fp8,0.007795199503501257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,10240,fp8,0.008008533219496409
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,8192,fp8,0.007705599814653396
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,7168,fp8,0.007744000355402628
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,6144,fp8,0.0073813334107399
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,5120,fp8,0.007435733824968339
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,4096,fp8,0.007116799801588058
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,3584,fp8,0.007479466994603475
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,3072,fp8,0.007235199958086014
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,2560,fp8,0.007480532924334208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,2048,fp8,0.006990932921568552
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,1536,fp8,0.007098666826883952
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,1024,fp8,0.0071520005663235995
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,768,fp8,0.00733440046509107
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,512,fp8,0.006938666353623072
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,256,fp8,0.006961066524187725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,128,fp8,0.0069514664510885865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,64,fp8,0.006858666241168976
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,32,fp8,0.006950400273005166
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,96,65536,fp8,0.010925867160161338
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,16384,fp8,0.007884799937407176
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,12288,fp8,0.007571200033028921
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,10240,fp8,0.008114133278528848
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,8192,fp8,0.007434666156768797
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,7168,fp8,0.007794133573770523
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,6144,fp8,0.007442132880290348
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,5120,fp8,0.007820800195137659
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,4096,fp8,0.00738666628797849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,3584,fp8,0.007470933596293132
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,3072,fp8,0.007082666456699371
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,2560,fp8,0.007253333181142808
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,2048,fp8,0.007074132810036342
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,1536,fp8,0.007143466422955196
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,1024,fp8,0.007320533196131389
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,768,fp8,0.007019733389218647
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,512,fp8,0.007117866228024164
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,256,fp8,0.006836267064015071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,128,fp8,0.006894932935635249
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,64,fp8,0.006791466722885767
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,32,fp8,0.006974933048089344
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,80,65536,fp8,0.010169600447018942
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,16384,fp8,0.00831573357184728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,12288,fp8,0.007482666273911794
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,10240,fp8,0.007616000374158224
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,8192,fp8,0.0075605330367883045
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,7168,fp8,0.007494399944941203
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,6144,fp8,0.007187200089295705
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,5120,fp8,0.007388799885908764
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,4096,fp8,0.007815467069546382
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,3584,fp8,0.007058132936557134
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,3072,fp8,0.007280000299215316
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,2560,fp8,0.00691306640704473
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,2048,fp8,0.007237333307663599
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,1536,fp8,0.00709226702650388
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,1024,fp8,0.007153066744407018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,768,fp8,0.006971733768781027
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,512,fp8,0.007171199470758439
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,256,fp8,0.006695466736952464
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,128,fp8,0.006983466446399689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,64,fp8,0.00674133375287056
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,32,fp8,0.006902400155862172
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,64,65536,fp8,0.009592533111572262
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,16384,fp8,0.007997866223255792
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,12288,fp8,0.007601066182057063
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,10240,fp8,0.007188266515731813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,8192,fp8,0.0074037335813045505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,7168,fp8,0.007165866593519847
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,6144,fp8,0.007265067100524903
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,5120,fp8,0.007092266529798508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,4096,fp8,0.007310933619737625
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,3584,fp8,0.0069749332964420315
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,3072,fp8,0.007089066257079443
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,2560,fp8,0.007032533238331477
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,2048,fp8,0.00730560024579366
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,1536,fp8,0.006904533257087071
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,1024,fp8,0.007140266646941502
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,768,fp8,0.006781866153081257
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,512,fp8,0.006963199873765309
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,256,fp8,0.006702933212121327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,128,fp8,0.0069045330087343865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,64,fp8,0.0068565328915913895
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,32,fp8,0.006720000257094701
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,48,65536,fp8,0.009053866565227508
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,16384,fp8,0.007153066744407018
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,12288,fp8,0.007352533688147862
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,10240,fp8,0.0070869334042072294
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,8192,fp8,0.007403733084599177
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,7168,fp8,0.007399466385444005
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,6144,fp8,0.007149866968393326
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,5120,fp8,0.00712000032265981
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,4096,fp8,0.007029333710670471
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,3584,fp8,0.007141333073377609
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,3072,fp8,0.006911999732255936
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,2560,fp8,0.007281066725651422
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,2048,fp8,0.0069877328972021745
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,1536,fp8,0.007188266267379126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,1024,fp8,0.007046399762233099
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,768,fp8,0.006742400427659352
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,512,fp8,0.006871466338634491
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,256,fp8,0.006755200276772181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,128,fp8,0.0069024001558621725
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,64,fp8,0.0066933331390221906
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,32,fp8,0.006851200262705486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,32,65536,fp8,0.00853973279396693
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,16384,fp8,0.007467733323574066
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,12288,fp8,0.007257599880297979
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,10240,fp8,0.007106133302052816
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,8192,fp8,0.006914133081833522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,7168,fp8,0.007108267148335776
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,6144,fp8,0.00691840002934138
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,5120,fp8,0.007350399593512218
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,4096,fp8,0.006902399907509486
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,3584,fp8,0.006847999989986419
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,3072,fp8,0.007189333438873291
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,2560,fp8,0.0068970667819182084
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,2048,fp8,0.006957866499821346
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,1536,fp8,0.006738133728504181
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,1024,fp8,0.00688213308652242
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,768,fp8,0.006617599974075953
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,512,fp8,0.006905600428581237
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,256,fp8,0.006782933572928111
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,128,fp8,0.006959999601046244
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,64,fp8,0.006782933076222738
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,32,fp8,0.006897066781918208
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,16,65536,fp8,0.008651733150084813
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,16384,fp8,0.007265066603819529
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,12288,fp8,0.007076266904671986
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,10240,fp8,0.00726080040136973
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,8192,fp8,0.00712320034702619
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,7168,fp8,0.007081599781910579
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,6144,fp8,0.0073749333620071415
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,5120,fp8,0.006906666606664657
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,4096,fp8,0.007118933647871017
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,3584,fp8,0.0072192003329594944
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,3072,fp8,0.007037866363922756
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,2560,fp8,0.00688533385594686
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,2048,fp8,0.006931199630101522
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,1536,fp8,0.006734933704137801
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,1024,fp8,0.007041066884994507
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,768,fp8,0.006781866649786632
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,512,fp8,0.006968533247709275
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,256,fp8,0.006741333256165186
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,128,fp8,0.00692586675286293
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,64,fp8,0.006705067058404288
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,32,fp8,0.006735999882221223
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,8,65536,fp8,0.007316266496976217
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,16384,fp8,0.006987733145554859
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,12288,fp8,0.007193600386381149
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,10240,fp8,0.00683733324209849
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,8192,fp8,0.007243733604749043
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,7168,fp8,0.006963199873765308
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,6144,fp8,0.007034666091203689
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,5120,fp8,0.00697173352042834
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,4096,fp8,0.0065152001877625786
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,3584,fp8,0.007028266539176305
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,3072,fp8,0.006849066416422527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,2560,fp8,0.006969600170850754
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,2048,fp8,0.006812799721956253
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,1536,fp8,0.0070271998643875126
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,1024,fp8,0.006814933319886527
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,768,fp8,0.00669866700967153
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,512,fp8,0.006940799454847972
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,256,fp8,0.006753066927194595
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,128,fp8,0.006954666475454967
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,64,fp8,0.0067317336797714225
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,32,fp8,0.006918400029341378
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,4,65536,fp8,0.007411200056473413
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,16384,fp8,0.006949333101511001
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,12288,fp8,0.006857599566380182
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,10240,fp8,0.007112533102432887
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,8192,fp8,0.0067904005448023485
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,7168,fp8,0.0070101338128248855
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,6144,fp8,0.006764799604813258
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,5120,fp8,0.007002666592597961
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,4096,fp8,0.006840533018112183
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,3584,fp8,0.007003733267386755
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,3072,fp8,0.007052800556023916
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,2560,fp8,0.006823466966549555
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,2048,fp8,0.007031467060248057
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,1536,fp8,0.006706132739782333
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,1024,fp8,0.0069514664510885865
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,768,fp8,0.006754133105278015
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,512,fp8,0.007001600414514542
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,256,fp8,0.006725333382685978
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,128,fp8,0.006825599819421768
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,64,fp8,0.006647466868162155
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,32,fp8,0.006908800452947617
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,2,65536,fp8,0.0069301329553127285
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,16384,fp8,0.007035733511050543
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,12288,fp8,0.006651733567317327
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,10240,fp8,0.006921599557002385
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,8192,fp8,0.006737066805362702
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,7168,fp8,0.006710399438937505
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,6144,fp8,0.0068842666844526924
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,5120,fp8,0.0066826663911342616
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,4096,fp8,0.00694720024863879
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,3584,fp8,0.006871466090281804
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,3072,fp8,0.006797866771618526
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,2560,fp8,0.006675200164318085
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,2048,fp8,0.006943999727567036
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,1536,fp8,0.006674133489529292
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,1024,fp8,0.006926933427651722
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,768,fp8,0.006657066692908606
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,512,fp8,0.006845866888761521
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,256,fp8,0.006654933343331019
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,128,fp8,0.006846933315197627
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,64,fp8,0.0067050668100515996
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,32,fp8,0.006827733665704728
TRTLLM,1.2.0rc5,NVIDIA H100 80GB HBM3,compute_scale,torch_ops,1,65536,fp8,0.0070485333601633705
